@@ -212,8 +212,10 @@ void av_frame_free(AVFrame **frame)
static int get_video_buffer(AVFrame *frame, int align)
{
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
- int ret, i, padded_height;
+ int ret, i, padded_height, total_size;
int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align);
+ ptrdiff_t linesizes[4];
+ size_t sizes[4];
if (!desc)
return AVERROR(EINVAL);
@@ -238,12 +240,22 @@ static int get_video_buffer(AVFrame *frame, int align)
frame->linesize[i] = FFALIGN(frame->linesize[i], align);
}
+ for (i = 0; i < 4; i++)
+ linesizes[i] = frame->linesize[i];
+
padded_height = FFALIGN(frame->height, 32);
- if ((ret = av_image_fill_pointers(frame->data, frame->format, padded_height,
- NULL, frame->linesize)) < 0)
+ if ((ret = av_image_fill_plane_sizes(sizes, frame->format,
+ padded_height, linesizes)) < 0)
return ret;
- frame->buf[0] = av_buffer_alloc(ret + 4*plane_padding);
+ total_size = 4*plane_padding;
+ for (i = 0; i < 4; i++) {
+ if (sizes[i] > INT_MAX - total_size)
+ return AVERROR(EINVAL);
+ total_size += sizes[i];
+ }
+
+ frame->buf[0] = av_buffer_alloc(total_size);
if (!frame->buf[0]) {
ret = AVERROR(ENOMEM);
goto fail;
This uses av_image_fill_plane_sizes instead of av_image_fill_pointers when we are getting plane sizes to avoid UB from adding offsets to NULL. Signed-off-by: Brian Kim <bkkim@google.com> --- libavutil/frame.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-)