diff mbox series

[FFmpeg-devel,v2,2/3] libavcodec/decode: avoid UB when getting plane sizes

Message ID CAO4ZO8dCyd9fD4eEyV3_FB=xZtbcL5WT-DM1tfmv4VPhTP7Q9Q@mail.gmail.com
State Accepted
Headers show
Series [FFmpeg-devel,v2,1/3] libavutil/imgutils: add utility to get plane sizes | expand

Checks

Context Check Description
andriy/default pending
andriy/make success Make finished
andriy/make_fate success Make fate finished

Commit Message

Brian Kim July 9, 2020, 1:51 a.m. UTC 
Patch attached
Subject: [PATCH 2/3] libavcodec/decode: avoid UB when getting plane sizes

This uses av_image_fill_plane_sizes instead of av_image_fill_pointers
when we are getting plane sizes to avoid UB from adding offsets to NULL.

Signed-off-by: Brian Kim <bkkim@google.com>
---
 libavcodec/decode.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index de9c079f9d..c1753cfffb 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -1471,12 +1471,12 @@  static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame)
 
     switch (avctx->codec_type) {
     case AVMEDIA_TYPE_VIDEO: {
-        uint8_t *data[4];
         int linesize[4];
-        int size[4] = { 0 };
         int w = frame->width;
         int h = frame->height;
-        int tmpsize, unaligned;
+        int unaligned;
+        ptrdiff_t tmpsize, linesize1[4];
+        size_t size[4];
 
         avcodec_align_dimensions2(avctx, &w, &h, pool->stride_align);
 
@@ -1494,20 +1494,22 @@  static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame)
                 unaligned |= linesize[i] % pool->stride_align[i];
         } while (unaligned);
 
-        tmpsize = av_image_fill_pointers(data, avctx->pix_fmt, h,
-                                         NULL, linesize);
+        for (i = 0; i < 4; i++)
+            linesize1[i] = linesize[i];
+        tmpsize = av_image_fill_plane_sizes(size, avctx->pix_fmt, h,
+                                            linesize1);
         if (tmpsize < 0) {
             ret = tmpsize;
             goto fail;
         }
 
-        for (i = 0; i < 3 && data[i + 1]; i++)
-            size[i] = data[i + 1] - data[i];
-        size[i] = tmpsize - (data[i] - data[0]);
-
         for (i = 0; i < 4; i++) {
             pool->linesize[i] = linesize[i];
             if (size[i]) {
+                if (size[i] > INT_MAX - (16 + STRIDE_ALIGN - 1)) {
+                    ret = AVERROR(EINVAL);
+                    goto fail;
+                }
                 pool->pools[i] = av_buffer_pool_init(size[i] + 16 + STRIDE_ALIGN - 1,
                                                      CONFIG_MEMORY_POISONING ?
                                                         NULL :