From patchwork Fri Apr 10 19:02:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: John Stebbins X-Patchwork-Id: 18816 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 9CE9044B994 for ; Fri, 10 Apr 2020 22:03:11 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 86B0B68B361; Fri, 10 Apr 2020 22:03:11 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mail.jetheaddev.com (mail.jetheaddev.com [70.164.99.34]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 8584E68B06A for ; Fri, 10 Apr 2020 22:03:04 +0300 (EEST) Received: from creator.alpe-d-promontory.fun (192.168.13.165) by cas.jetheaddev.com (192.168.13.27) with Microsoft SMTP Server (TLS) id 14.3.351.0; Fri, 10 Apr 2020 12:03:03 -0700 From: John Stebbins To: Date: Fri, 10 Apr 2020 13:02:17 -0600 Message-ID: <20200410190223.134413-2-jstebbins@jetheaddev.com> X-Mailer: git-send-email 2.25.2 In-Reply-To: <20200410190223.134413-1-jstebbins@jetheaddev.com> References: <20200410190223.134413-1-jstebbins@jetheaddev.com> MIME-Version: 1.0 X-Originating-IP: [192.168.13.165] Subject: [FFmpeg-devel] [PATCH 1/7] lavc/dvbsub: fix potential encode buffer overflow X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" encode buffer size was ignored --- libavcodec/dvbsub.c | 97 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/libavcodec/dvbsub.c b/libavcodec/dvbsub.c index a8d43d81d6..a0be0b1056 100644 --- a/libavcodec/dvbsub.c +++ b/libavcodec/dvbsub.c @@ -37,11 +37,11 @@ typedef struct DVBSubtitleContext { }\ } -static void dvb_encode_rle2(uint8_t **pq, - const uint8_t *bitmap, int linesize, - int w, int h) +static int dvb_encode_rle2(uint8_t **pq, int buf_size, + const uint8_t *bitmap, int linesize, + int w, int h) { - uint8_t *q; + uint8_t *q, *line_begin; unsigned int bitbuf; int bitcnt; int x, y, len, x1, v, color; @@ -49,6 +49,10 @@ static void dvb_encode_rle2(uint8_t **pq, q = *pq; for(y = 0; y < h; y++) { + // Worst case line is 3 bits per value + 4 bytes overhead + if (buf_size * 8 < w * 3 + 32) + return AVERROR_BUFFER_TOO_SMALL; + line_begin = q; *q++ = 0x10; bitbuf = 0; bitcnt = 6; @@ -109,8 +113,11 @@ static void dvb_encode_rle2(uint8_t **pq, } *q++ = 0xf0; bitmap += linesize; + buf_size -= q - line_begin; } + len = q - *pq; *pq = q; + return len; } #define PUTBITS4(val)\ @@ -125,11 +132,11 @@ static void dvb_encode_rle2(uint8_t **pq, } /* some DVB decoders only implement 4 bits/pixel */ -static void dvb_encode_rle4(uint8_t **pq, - const uint8_t *bitmap, int linesize, - int w, int h) +static int dvb_encode_rle4(uint8_t **pq, int buf_size, + const uint8_t *bitmap, int linesize, + int w, int h) { - uint8_t *q; + uint8_t *q, *line_begin; unsigned int bitbuf; int bitcnt; int x, y, len, x1, v, color; @@ -137,6 +144,10 @@ static void dvb_encode_rle4(uint8_t **pq, q = *pq; for(y = 0; y < h; y++) { + // Worst case line is 6 bits per value, + 4 bytes overhead + if (buf_size * 8 < w * 6 + 32) + return AVERROR_BUFFER_TOO_SMALL; + line_begin = q; *q++ = 0x11; bitbuf = 0; bitcnt = 4; @@ -189,20 +200,27 @@ static void dvb_encode_rle4(uint8_t **pq, } *q++ = 0xf0; bitmap += linesize; + buf_size -= q - line_begin; } + len = q - *pq; *pq = q; + return len; } -static void dvb_encode_rle8(uint8_t **pq, - const uint8_t *bitmap, int linesize, - int w, int h) +static int dvb_encode_rle8(uint8_t **pq, int buf_size, + const uint8_t *bitmap, int linesize, + int w, int h) { - uint8_t *q; + uint8_t *q, *line_begin; int x, y, len, x1, color; q = *pq; for (y = 0; y < h; y++) { + // Worst case line is 12 bits per value, + 3 bytes overhead + if (buf_size * 8 < w * 12 + 24) + return AVERROR_BUFFER_TOO_SMALL; + line_begin = q; *q++ = 0x12; x = 0; @@ -243,12 +261,16 @@ static void dvb_encode_rle8(uint8_t **pq, *q++ = 0x00; *q++ = 0xf0; bitmap += linesize; + buf_size -= q - line_begin; } + len = q - *pq; *pq = q; + return len; } static int encode_dvb_subtitles(AVCodecContext *avctx, - uint8_t *outbuf, const AVSubtitle *h) + uint8_t *outbuf, int buf_size, + const AVSubtitle *h) { DVBSubtitleContext *s = avctx->priv_data; uint8_t *q, *pseg_len; @@ -263,6 +285,8 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, return -1; if (avctx->width > 0 && avctx->height > 0) { + if (buf_size < 11) + return AVERROR_BUFFER_TOO_SMALL; /* display definition segment */ *q++ = 0x0f; /* sync_byte */ *q++ = 0x14; /* segment_type */ @@ -273,10 +297,13 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, bytestream_put_be16(&q, avctx->width - 1); /* display width */ bytestream_put_be16(&q, avctx->height - 1); /* display height */ bytestream_put_be16(&pseg_len, q - pseg_len - 2); + buf_size -= 11; } /* page composition segment */ + if (buf_size < 8 + h->num_rects * 6) + return AVERROR_BUFFER_TOO_SMALL; *q++ = 0x0f; /* sync_byte */ *q++ = 0x10; /* segment_type */ bytestream_put_be16(&q, page_id); @@ -295,9 +322,12 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, } bytestream_put_be16(&pseg_len, q - pseg_len - 2); + buf_size -= 8 + h->num_rects * 6; if (h->num_rects) { for (clut_id = 0; clut_id < h->num_rects; clut_id++) { + if (buf_size < 6 + h->rects[clut_id]->nb_colors * 6) + return AVERROR_BUFFER_TOO_SMALL; /* CLUT segment */ @@ -343,9 +373,12 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, } bytestream_put_be16(&pseg_len, q - pseg_len - 2); + buf_size -= 6 + h->rects[clut_id]->nb_colors * 6; } } + if (buf_size < h->num_rects * 22) + return AVERROR_BUFFER_TOO_SMALL; for (region_id = 0; region_id < h->num_rects; region_id++) { /* region composition segment */ @@ -385,13 +418,17 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, bytestream_put_be16(&pseg_len, q - pseg_len - 2); } + buf_size -= h->num_rects * 22; if (h->num_rects) { for (object_id = 0; object_id < h->num_rects; object_id++) { - void (*dvb_encode_rle)(uint8_t **pq, - const uint8_t *bitmap, int linesize, - int w, int h); + int (*dvb_encode_rle)(uint8_t **pq, int buf_size, + const uint8_t *bitmap, int linesize, + int w, int h); + + if (buf_size < 13) + return AVERROR_BUFFER_TOO_SMALL; /* bpp_index maths */ if (h->rects[object_id]->nb_colors <= 4) { @@ -420,19 +457,32 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, non_modifying_color_flag */ { uint8_t *ptop_field_len, *pbottom_field_len, *top_ptr, *bottom_ptr; + int ret; ptop_field_len = q; q += 2; pbottom_field_len = q; q += 2; + buf_size -= 13; top_ptr = q; - dvb_encode_rle(&q, h->rects[object_id]->data[0], h->rects[object_id]->w * 2, - h->rects[object_id]->w, h->rects[object_id]->h >> 1); + ret = dvb_encode_rle(&q, buf_size, + h->rects[object_id]->data[0], + h->rects[object_id]->w * 2, + h->rects[object_id]->w, + h->rects[object_id]->h >> 1); + if (ret < 0) + return ret; + buf_size -= ret; bottom_ptr = q; - dvb_encode_rle(&q, h->rects[object_id]->data[0] + h->rects[object_id]->w, - h->rects[object_id]->w * 2, h->rects[object_id]->w, - h->rects[object_id]->h >> 1); + ret = dvb_encode_rle(&q, buf_size, + h->rects[object_id]->data[0] + h->rects[object_id]->w, + h->rects[object_id]->w * 2, + h->rects[object_id]->w, + h->rects[object_id]->h >> 1); + if (ret < 0) + return ret; + buf_size -= ret; bytestream_put_be16(&ptop_field_len, bottom_ptr - top_ptr); bytestream_put_be16(&pbottom_field_len, q - bottom_ptr); @@ -444,6 +494,8 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, /* end of display set segment */ + if (buf_size < 6) + return AVERROR_BUFFER_TOO_SMALL; *q++ = 0x0f; /* sync_byte */ *q++ = 0x80; /* segment_type */ bytestream_put_be16(&q, page_id); @@ -451,6 +503,7 @@ static int encode_dvb_subtitles(AVCodecContext *avctx, q += 2; /* segment length */ bytestream_put_be16(&pseg_len, q - pseg_len - 2); + buf_size -= 6; s->object_version = (s->object_version + 1) & 0xf; return q - outbuf; @@ -462,7 +515,7 @@ static int dvbsub_encode(AVCodecContext *avctx, { int ret; - ret = encode_dvb_subtitles(avctx, buf, sub); + ret = encode_dvb_subtitles(avctx, buf, buf_size, sub); return ret; }