[FFmpeg-devel,4/6] avcodec/flicvideo: Make line_packets int

Submitted by Michael Niedermayer on June 21, 2019, 11:29 p.m.

Details

Message ID 20190621232936.11052-4-michael@niedermayer.cc
State New
Headers show

Commit Message

Michael Niedermayer June 21, 2019, 11:29 p.m.
Fixes: signed integer overflow: -32768 * 196032 cannot be represented in type 'int'
Fixes: 15300/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_FLIC_fuzzer-5733319519502336

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/flicvideo.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

Comments

Paul B Mahol June 22, 2019, 2:58 p.m.
On 6/22/19, Michael Niedermayer <michael@niedermayer.cc> wrote:
> Fixes: signed integer overflow: -32768 * 196032 cannot be represented in
> type 'int'
> Fixes:
> 15300/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_FLIC_fuzzer-5733319519502336
>
> Found-by: continuous fuzzing process
> https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> ---
>  libavcodec/flicvideo.c | 14 +++++++-------
>  1 file changed, 7 insertions(+), 7 deletions(-)
>
> diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
> index ba5bda48c4..cd9cd089af 100644
> --- a/libavcodec/flicvideo.c
> +++ b/libavcodec/flicvideo.c
> @@ -175,7 +175,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
>      int lines;
>      int compressed_lines;
>      int starting_line;
> -    signed short line_packets;
> +    int line_packets;
>      int y_ptr;
>      int byte_run;
>      int pixel_skip;
> @@ -274,7 +274,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
>                      break;
>                  if (y_ptr > pixel_limit)
>                      return AVERROR_INVALIDDATA;
> -                line_packets = bytestream2_get_le16(&g2);
> +                line_packets = (int16_t)bytestream2_get_le16(&g2);
>                  if ((line_packets & 0xC000) == 0xC000) {
>                      // line skip opcode
>                      line_packets = -line_packets;
> @@ -340,7 +340,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
>                  pixel_countdown = s->avctx->width;
>                  if (bytestream2_tell(&g2) + 1 > stream_ptr_after_chunk)
>                      break;
> -                line_packets = bytestream2_get_byte(&g2);
> +                line_packets = (int16_t)bytestream2_get_byte(&g2);
>                  if (line_packets > 0) {
>                      for (i = 0; i < line_packets; i++) {
>                          /* account for the skip bytes */
> @@ -508,7 +508,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext
> *avctx,
>
>      int lines;
>      int compressed_lines;
> -    signed short line_packets;
> +    int line_packets;
>      int y_ptr;
>      int byte_run;
>      int pixel_skip;
> @@ -572,7 +572,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext
> *avctx,
>                      break;
>                  if (y_ptr > pixel_limit)
>                      return AVERROR_INVALIDDATA;
> -                line_packets = bytestream2_get_le16(&g2);
> +                line_packets = (int16_t)bytestream2_get_le16(&g2);
>                  if (line_packets < 0) {
>                      line_packets = -line_packets;
>                      if (line_packets > s->avctx->height)
> @@ -806,7 +806,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
> *avctx,
>
>      int lines;
>      int compressed_lines;
> -    signed short line_packets;
> +    int line_packets;
>      int y_ptr;
>      int byte_run;
>      int pixel_skip;
> @@ -870,7 +870,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
> *avctx,
>                      break;
>                  if (y_ptr > pixel_limit)
>                      return AVERROR_INVALIDDATA;
> -                line_packets = bytestream2_get_le16(&g2);
> +                line_packets = (int16_t)bytestream2_get_le16(&g2);
>                  if (line_packets < 0) {
>                      line_packets = -line_packets;
>                      if (line_packets > s->avctx->height)
> --
> 2.22.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

In some cases casting in not needed.
Also cant you use sign_extend ?
Michael Niedermayer July 8, 2019, 6:51 a.m.
On Sat, Jun 22, 2019 at 04:58:37PM +0200, Paul B Mahol wrote:
> On 6/22/19, Michael Niedermayer <michael@niedermayer.cc> wrote:
> > Fixes: signed integer overflow: -32768 * 196032 cannot be represented in
> > type 'int'
> > Fixes:
> > 15300/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_FLIC_fuzzer-5733319519502336
> >
> > Found-by: continuous fuzzing process
> > https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> > ---
> >  libavcodec/flicvideo.c | 14 +++++++-------
> >  1 file changed, 7 insertions(+), 7 deletions(-)
> >
> > diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
> > index ba5bda48c4..cd9cd089af 100644
> > --- a/libavcodec/flicvideo.c
> > +++ b/libavcodec/flicvideo.c
> > @@ -175,7 +175,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
> >      int lines;
> >      int compressed_lines;
> >      int starting_line;
> > -    signed short line_packets;
> > +    int line_packets;
> >      int y_ptr;
> >      int byte_run;
> >      int pixel_skip;
> > @@ -274,7 +274,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
> >                      break;
> >                  if (y_ptr > pixel_limit)
> >                      return AVERROR_INVALIDDATA;
> > -                line_packets = bytestream2_get_le16(&g2);
> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
> >                  if ((line_packets & 0xC000) == 0xC000) {
> >                      // line skip opcode
> >                      line_packets = -line_packets;
> > @@ -340,7 +340,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
> >                  pixel_countdown = s->avctx->width;
> >                  if (bytestream2_tell(&g2) + 1 > stream_ptr_after_chunk)
> >                      break;
> > -                line_packets = bytestream2_get_byte(&g2);
> > +                line_packets = (int16_t)bytestream2_get_byte(&g2);
> >                  if (line_packets > 0) {
> >                      for (i = 0; i < line_packets; i++) {
> >                          /* account for the skip bytes */
> > @@ -508,7 +508,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext
> > *avctx,
> >
> >      int lines;
> >      int compressed_lines;
> > -    signed short line_packets;
> > +    int line_packets;
> >      int y_ptr;
> >      int byte_run;
> >      int pixel_skip;
> > @@ -572,7 +572,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext
> > *avctx,
> >                      break;
> >                  if (y_ptr > pixel_limit)
> >                      return AVERROR_INVALIDDATA;
> > -                line_packets = bytestream2_get_le16(&g2);
> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
> >                  if (line_packets < 0) {
> >                      line_packets = -line_packets;
> >                      if (line_packets > s->avctx->height)
> > @@ -806,7 +806,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
> > *avctx,
> >
> >      int lines;
> >      int compressed_lines;
> > -    signed short line_packets;
> > +    int line_packets;
> >      int y_ptr;
> >      int byte_run;
> >      int pixel_skip;
> > @@ -870,7 +870,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
> > *avctx,
> >                      break;
> >                  if (y_ptr > pixel_limit)
> >                      return AVERROR_INVALIDDATA;
> > -                line_packets = bytestream2_get_le16(&g2);
> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
> >                  if (line_packets < 0) {
> >                      line_packets = -line_packets;
> >                      if (line_packets > s->avctx->height)
> > --
> > 2.22.0
> >
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> 
> In some cases casting in not needed.

unneeded one dropped


> Also cant you use sign_extend ?

certainly but that might be slower.
Do you prefer if i use sign_extend ?

thanks

[...]
Paul B Mahol July 8, 2019, 7:04 a.m.
On 7/8/19, Michael Niedermayer <michael@niedermayer.cc> wrote:
> On Sat, Jun 22, 2019 at 04:58:37PM +0200, Paul B Mahol wrote:
>> On 6/22/19, Michael Niedermayer <michael@niedermayer.cc> wrote:
>> > Fixes: signed integer overflow: -32768 * 196032 cannot be represented
>> > in
>> > type 'int'
>> > Fixes:
>> > 15300/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_FLIC_fuzzer-5733319519502336
>> >
>> > Found-by: continuous fuzzing process
>> > https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
>> > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
>> > ---
>> >  libavcodec/flicvideo.c | 14 +++++++-------
>> >  1 file changed, 7 insertions(+), 7 deletions(-)
>> >
>> > diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
>> > index ba5bda48c4..cd9cd089af 100644
>> > --- a/libavcodec/flicvideo.c
>> > +++ b/libavcodec/flicvideo.c
>> > @@ -175,7 +175,7 @@ static int flic_decode_frame_8BPP(AVCodecContext
>> > *avctx,
>> >      int lines;
>> >      int compressed_lines;
>> >      int starting_line;
>> > -    signed short line_packets;
>> > +    int line_packets;
>> >      int y_ptr;
>> >      int byte_run;
>> >      int pixel_skip;
>> > @@ -274,7 +274,7 @@ static int flic_decode_frame_8BPP(AVCodecContext
>> > *avctx,
>> >                      break;
>> >                  if (y_ptr > pixel_limit)
>> >                      return AVERROR_INVALIDDATA;
>> > -                line_packets = bytestream2_get_le16(&g2);
>> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
>> >                  if ((line_packets & 0xC000) == 0xC000) {
>> >                      // line skip opcode
>> >                      line_packets = -line_packets;
>> > @@ -340,7 +340,7 @@ static int flic_decode_frame_8BPP(AVCodecContext
>> > *avctx,
>> >                  pixel_countdown = s->avctx->width;
>> >                  if (bytestream2_tell(&g2) + 1 >
>> > stream_ptr_after_chunk)
>> >                      break;
>> > -                line_packets = bytestream2_get_byte(&g2);
>> > +                line_packets = (int16_t)bytestream2_get_byte(&g2);
>> >                  if (line_packets > 0) {
>> >                      for (i = 0; i < line_packets; i++) {
>> >                          /* account for the skip bytes */
>> > @@ -508,7 +508,7 @@ static int
>> > flic_decode_frame_15_16BPP(AVCodecContext
>> > *avctx,
>> >
>> >      int lines;
>> >      int compressed_lines;
>> > -    signed short line_packets;
>> > +    int line_packets;
>> >      int y_ptr;
>> >      int byte_run;
>> >      int pixel_skip;
>> > @@ -572,7 +572,7 @@ static int
>> > flic_decode_frame_15_16BPP(AVCodecContext
>> > *avctx,
>> >                      break;
>> >                  if (y_ptr > pixel_limit)
>> >                      return AVERROR_INVALIDDATA;
>> > -                line_packets = bytestream2_get_le16(&g2);
>> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
>> >                  if (line_packets < 0) {
>> >                      line_packets = -line_packets;
>> >                      if (line_packets > s->avctx->height)
>> > @@ -806,7 +806,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
>> > *avctx,
>> >
>> >      int lines;
>> >      int compressed_lines;
>> > -    signed short line_packets;
>> > +    int line_packets;
>> >      int y_ptr;
>> >      int byte_run;
>> >      int pixel_skip;
>> > @@ -870,7 +870,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
>> > *avctx,
>> >                      break;
>> >                  if (y_ptr > pixel_limit)
>> >                      return AVERROR_INVALIDDATA;
>> > -                line_packets = bytestream2_get_le16(&g2);
>> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
>> >                  if (line_packets < 0) {
>> >                      line_packets = -line_packets;
>> >                      if (line_packets > s->avctx->height)
>> > --
>> > 2.22.0
>> >
>> > _______________________________________________
>> > ffmpeg-devel mailing list
>> > ffmpeg-devel@ffmpeg.org
>> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>> >
>> > To unsubscribe, visit link above, or email
>> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>>
>> In some cases casting in not needed.
>
> unneeded one dropped
>
>
>> Also cant you use sign_extend ?
>
> certainly but that might be slower.
> Do you prefer if i use sign_extend ?

Not if it is slower.

>
> thanks
>
> [...]
> --
> Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> Everything should be made as simple as possible, but not simpler.
> -- Albert Einstein
>
Michael Niedermayer July 21, 2019, 8:46 a.m.
On Mon, Jul 08, 2019 at 09:04:34AM +0200, Paul B Mahol wrote:
> On 7/8/19, Michael Niedermayer <michael@niedermayer.cc> wrote:
> > On Sat, Jun 22, 2019 at 04:58:37PM +0200, Paul B Mahol wrote:
> >> On 6/22/19, Michael Niedermayer <michael@niedermayer.cc> wrote:
> >> > Fixes: signed integer overflow: -32768 * 196032 cannot be represented
> >> > in
> >> > type 'int'
> >> > Fixes:
> >> > 15300/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_FLIC_fuzzer-5733319519502336
> >> >
> >> > Found-by: continuous fuzzing process
> >> > https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> >> > Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
> >> > ---
> >> >  libavcodec/flicvideo.c | 14 +++++++-------
> >> >  1 file changed, 7 insertions(+), 7 deletions(-)
> >> >
> >> > diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
> >> > index ba5bda48c4..cd9cd089af 100644
> >> > --- a/libavcodec/flicvideo.c
> >> > +++ b/libavcodec/flicvideo.c
> >> > @@ -175,7 +175,7 @@ static int flic_decode_frame_8BPP(AVCodecContext
> >> > *avctx,
> >> >      int lines;
> >> >      int compressed_lines;
> >> >      int starting_line;
> >> > -    signed short line_packets;
> >> > +    int line_packets;
> >> >      int y_ptr;
> >> >      int byte_run;
> >> >      int pixel_skip;
> >> > @@ -274,7 +274,7 @@ static int flic_decode_frame_8BPP(AVCodecContext
> >> > *avctx,
> >> >                      break;
> >> >                  if (y_ptr > pixel_limit)
> >> >                      return AVERROR_INVALIDDATA;
> >> > -                line_packets = bytestream2_get_le16(&g2);
> >> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
> >> >                  if ((line_packets & 0xC000) == 0xC000) {
> >> >                      // line skip opcode
> >> >                      line_packets = -line_packets;
> >> > @@ -340,7 +340,7 @@ static int flic_decode_frame_8BPP(AVCodecContext
> >> > *avctx,
> >> >                  pixel_countdown = s->avctx->width;
> >> >                  if (bytestream2_tell(&g2) + 1 >
> >> > stream_ptr_after_chunk)
> >> >                      break;
> >> > -                line_packets = bytestream2_get_byte(&g2);
> >> > +                line_packets = (int16_t)bytestream2_get_byte(&g2);
> >> >                  if (line_packets > 0) {
> >> >                      for (i = 0; i < line_packets; i++) {
> >> >                          /* account for the skip bytes */
> >> > @@ -508,7 +508,7 @@ static int
> >> > flic_decode_frame_15_16BPP(AVCodecContext
> >> > *avctx,
> >> >
> >> >      int lines;
> >> >      int compressed_lines;
> >> > -    signed short line_packets;
> >> > +    int line_packets;
> >> >      int y_ptr;
> >> >      int byte_run;
> >> >      int pixel_skip;
> >> > @@ -572,7 +572,7 @@ static int
> >> > flic_decode_frame_15_16BPP(AVCodecContext
> >> > *avctx,
> >> >                      break;
> >> >                  if (y_ptr > pixel_limit)
> >> >                      return AVERROR_INVALIDDATA;
> >> > -                line_packets = bytestream2_get_le16(&g2);
> >> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
> >> >                  if (line_packets < 0) {
> >> >                      line_packets = -line_packets;
> >> >                      if (line_packets > s->avctx->height)
> >> > @@ -806,7 +806,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
> >> > *avctx,
> >> >
> >> >      int lines;
> >> >      int compressed_lines;
> >> > -    signed short line_packets;
> >> > +    int line_packets;
> >> >      int y_ptr;
> >> >      int byte_run;
> >> >      int pixel_skip;
> >> > @@ -870,7 +870,7 @@ static int flic_decode_frame_24BPP(AVCodecContext
> >> > *avctx,
> >> >                      break;
> >> >                  if (y_ptr > pixel_limit)
> >> >                      return AVERROR_INVALIDDATA;
> >> > -                line_packets = bytestream2_get_le16(&g2);
> >> > +                line_packets = (int16_t)bytestream2_get_le16(&g2);
> >> >                  if (line_packets < 0) {
> >> >                      line_packets = -line_packets;
> >> >                      if (line_packets > s->avctx->height)
> >> > --
> >> > 2.22.0
> >> >
> >> > _______________________________________________
> >> > ffmpeg-devel mailing list
> >> > ffmpeg-devel@ffmpeg.org
> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >> >
> >> > To unsubscribe, visit link above, or email
> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> >>
> >> In some cases casting in not needed.
> >
> > unneeded one dropped
> >
> >
> >> Also cant you use sign_extend ?
> >
> > certainly but that might be slower.
> > Do you prefer if i use sign_extend ?
> 
> Not if it is slower.

tested, no speedloss, will apply with sign_extend()

thanks

[...]

Patch hide | download patch | download mbox

diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
index ba5bda48c4..cd9cd089af 100644
--- a/libavcodec/flicvideo.c
+++ b/libavcodec/flicvideo.c
@@ -175,7 +175,7 @@  static int flic_decode_frame_8BPP(AVCodecContext *avctx,
     int lines;
     int compressed_lines;
     int starting_line;
-    signed short line_packets;
+    int line_packets;
     int y_ptr;
     int byte_run;
     int pixel_skip;
@@ -274,7 +274,7 @@  static int flic_decode_frame_8BPP(AVCodecContext *avctx,
                     break;
                 if (y_ptr > pixel_limit)
                     return AVERROR_INVALIDDATA;
-                line_packets = bytestream2_get_le16(&g2);
+                line_packets = (int16_t)bytestream2_get_le16(&g2);
                 if ((line_packets & 0xC000) == 0xC000) {
                     // line skip opcode
                     line_packets = -line_packets;
@@ -340,7 +340,7 @@  static int flic_decode_frame_8BPP(AVCodecContext *avctx,
                 pixel_countdown = s->avctx->width;
                 if (bytestream2_tell(&g2) + 1 > stream_ptr_after_chunk)
                     break;
-                line_packets = bytestream2_get_byte(&g2);
+                line_packets = (int16_t)bytestream2_get_byte(&g2);
                 if (line_packets > 0) {
                     for (i = 0; i < line_packets; i++) {
                         /* account for the skip bytes */
@@ -508,7 +508,7 @@  static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
 
     int lines;
     int compressed_lines;
-    signed short line_packets;
+    int line_packets;
     int y_ptr;
     int byte_run;
     int pixel_skip;
@@ -572,7 +572,7 @@  static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
                     break;
                 if (y_ptr > pixel_limit)
                     return AVERROR_INVALIDDATA;
-                line_packets = bytestream2_get_le16(&g2);
+                line_packets = (int16_t)bytestream2_get_le16(&g2);
                 if (line_packets < 0) {
                     line_packets = -line_packets;
                     if (line_packets > s->avctx->height)
@@ -806,7 +806,7 @@  static int flic_decode_frame_24BPP(AVCodecContext *avctx,
 
     int lines;
     int compressed_lines;
-    signed short line_packets;
+    int line_packets;
     int y_ptr;
     int byte_run;
     int pixel_skip;
@@ -870,7 +870,7 @@  static int flic_decode_frame_24BPP(AVCodecContext *avctx,
                     break;
                 if (y_ptr > pixel_limit)
                     return AVERROR_INVALIDDATA;
-                line_packets = bytestream2_get_le16(&g2);
+                line_packets = (int16_t)bytestream2_get_le16(&g2);
                 if (line_packets < 0) {
                     line_packets = -line_packets;
                     if (line_packets > s->avctx->height)