From patchwork Wed May 3 03:21:51 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael Niedermayer X-Patchwork-Id: 3565 Delivered-To: ffmpegpatchwork@gmail.com Received: by 10.103.3.129 with SMTP id 123csp2182696vsd; Tue, 2 May 2017 20:22:05 -0700 (PDT) X-Received: by 10.28.54.151 with SMTP id y23mr4797889wmh.90.1493781725351; Tue, 02 May 2017 20:22:05 -0700 (PDT) Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org. [79.124.17.100]) by mx.google.com with ESMTP id 45si11770878wry.54.2017.05.02.20.22.04; Tue, 02 May 2017 20:22:05 -0700 (PDT) Received-SPF: pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) client-ip=79.124.17.100; Authentication-Results: mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 23E9F6882B9; Wed, 3 May 2017 06:21:58 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from vie01a-dmta-pe04-1.mx.upcmail.net (vie01a-dmta-pe04-1.mx.upcmail.net [62.179.121.163]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 7AB4668090C for ; Wed, 3 May 2017 06:21:51 +0300 (EEST) Received: from [172.31.216.43] (helo=vie01a-pemc-psmtp-pe01) by vie01a-dmta-pe04.mx.upcmail.net with esmtp (Exim 4.88) (envelope-from ) id 1d5krS-0004Nn-Kj for ffmpeg-devel@ffmpeg.org; Wed, 03 May 2017 05:21:54 +0200 Received: from localhost ([213.47.41.20]) by vie01a-pemc-psmtp-pe01 with SMTP @ mailcloud.upcmail.net id FfMt1v00p0S5wYM01fMu6Q; Wed, 03 May 2017 05:21:54 +0200 X-SourceIP: 213.47.41.20 From: Michael Niedermayer To: FFmpeg development discussions and patches Date: Wed, 3 May 2017 05:21:51 +0200 Message-Id: <20170503032151.25225-2-michael@niedermayer.cc> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20170503032151.25225-1-michael@niedermayer.cc> References: <20170503032151.25225-1-michael@niedermayer.cc> Subject: [FFmpeg-devel] [PATCH 2/2] libavcodec/mpeg4videodec: Convert sprite_offset to 64bit X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" This avoids intermediates from overflowing (the final values are checked) Fixes: runtime error: signed integer overflow: -167712 + -2147352576 cannot be represented in type 'int' Fixes: 1298/clusterfuzz-testcase-minimized-5955580877340672 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/mpeg4videodec.c | 102 ++++++++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c index 791a07bbbb..39f177f8d0 100644 --- a/libavcodec/mpeg4videodec.c +++ b/libavcodec/mpeg4videodec.c @@ -178,6 +178,7 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g int min_ab, i, w2, h2, w3, h3; int sprite_ref[4][2]; int virtual_ref[2][2]; + int64_t sprite_offset[2][2]; // only true for rectangle shapes const int vop_ref[4][2] = { { 0, 0 }, { s->width, 0 }, @@ -257,10 +258,10 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g switch (ctx->num_sprite_warping_points) { case 0: - s->sprite_offset[0][0] = - s->sprite_offset[0][1] = - s->sprite_offset[1][0] = - s->sprite_offset[1][1] = 0; + sprite_offset[0][0] = + sprite_offset[0][1] = + sprite_offset[1][0] = + sprite_offset[1][1] = 0; s->sprite_delta[0][0] = a; s->sprite_delta[0][1] = s->sprite_delta[1][0] = 0; @@ -269,11 +270,11 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g ctx->sprite_shift[1] = 0; break; case 1: // GMC only - s->sprite_offset[0][0] = sprite_ref[0][0] - a * vop_ref[0][0]; - s->sprite_offset[0][1] = sprite_ref[0][1] - a * vop_ref[0][1]; - s->sprite_offset[1][0] = ((sprite_ref[0][0] >> 1) | (sprite_ref[0][0] & 1)) - + sprite_offset[0][0] = sprite_ref[0][0] - a * vop_ref[0][0]; + sprite_offset[0][1] = sprite_ref[0][1] - a * vop_ref[0][1]; + sprite_offset[1][0] = ((sprite_ref[0][0] >> 1) | (sprite_ref[0][0] & 1)) - a * (vop_ref[0][0] / 2); - s->sprite_offset[1][1] = ((sprite_ref[0][1] >> 1) | (sprite_ref[0][1] & 1)) - + sprite_offset[1][1] = ((sprite_ref[0][1] >> 1) | (sprite_ref[0][1] & 1)) - a * (vop_ref[0][1] / 2); s->sprite_delta[0][0] = a; s->sprite_delta[0][1] = @@ -283,22 +284,22 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g ctx->sprite_shift[1] = 0; break; case 2: - s->sprite_offset[0][0] = (sprite_ref[0][0] * (1 << alpha + rho)) + + sprite_offset[0][0] = (sprite_ref[0][0] * (1 << alpha + rho)) + (-r * sprite_ref[0][0] + virtual_ref[0][0]) * (-vop_ref[0][0]) + (r * sprite_ref[0][1] - virtual_ref[0][1]) * (-vop_ref[0][1]) + (1 << (alpha + rho - 1)); - s->sprite_offset[0][1] = (sprite_ref[0][1] * (1 << alpha + rho)) + + sprite_offset[0][1] = (sprite_ref[0][1] * (1 << alpha + rho)) + (-r * sprite_ref[0][1] + virtual_ref[0][1]) * (-vop_ref[0][0]) + (-r * sprite_ref[0][0] + virtual_ref[0][0]) * (-vop_ref[0][1]) + (1 << (alpha + rho - 1)); - s->sprite_offset[1][0] = ((-r * sprite_ref[0][0] + virtual_ref[0][0]) * + sprite_offset[1][0] = ((-r * sprite_ref[0][0] + virtual_ref[0][0]) * (-2 * vop_ref[0][0] + 1) + (r * sprite_ref[0][1] - virtual_ref[0][1]) * (-2 * vop_ref[0][1] + 1) + 2 * w2 * r * sprite_ref[0][0] - 16 * w2 + (1 << (alpha + rho + 1))); - s->sprite_offset[1][1] = ((-r * sprite_ref[0][1] + virtual_ref[0][1]) * + sprite_offset[1][1] = ((-r * sprite_ref[0][1] + virtual_ref[0][1]) * (-2 * vop_ref[0][0] + 1) + (-r * sprite_ref[0][0] + virtual_ref[0][0]) * (-2 * vop_ref[0][1] + 1) + 2 * w2 * r * @@ -315,30 +316,22 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g min_ab = FFMIN(alpha, beta); w3 = w2 >> min_ab; h3 = h2 >> min_ab; - s->sprite_offset[0][0] = (sprite_ref[0][0] * (1<<(alpha + beta + rho - min_ab))) + - (-r * sprite_ref[0][0] + virtual_ref[0][0]) * - h3 * (-vop_ref[0][0]) + - (-r * sprite_ref[0][0] + virtual_ref[1][0]) * - w3 * (-vop_ref[0][1]) + - (1 << (alpha + beta + rho - min_ab - 1)); - s->sprite_offset[0][1] = (sprite_ref[0][1] * (1 << (alpha + beta + rho - min_ab))) + - (-r * sprite_ref[0][1] + virtual_ref[0][1]) * - h3 * (-vop_ref[0][0]) + - (-r * sprite_ref[0][1] + virtual_ref[1][1]) * - w3 * (-vop_ref[0][1]) + - (1 << (alpha + beta + rho - min_ab - 1)); - s->sprite_offset[1][0] = (-r * sprite_ref[0][0] + virtual_ref[0][0]) * - h3 * (-2 * vop_ref[0][0] + 1) + - (-r * sprite_ref[0][0] + virtual_ref[1][0]) * - w3 * (-2 * vop_ref[0][1] + 1) + 2 * w2 * h3 * - r * sprite_ref[0][0] - 16 * w2 * h3 + - (1 << (alpha + beta + rho - min_ab + 1)); - s->sprite_offset[1][1] = (-r * sprite_ref[0][1] + virtual_ref[0][1]) * - h3 * (-2 * vop_ref[0][0] + 1) + - (-r * sprite_ref[0][1] + virtual_ref[1][1]) * - w3 * (-2 * vop_ref[0][1] + 1) + 2 * w2 * h3 * - r * sprite_ref[0][1] - 16 * w2 * h3 + - (1 << (alpha + beta + rho - min_ab + 1)); + sprite_offset[0][0] = ((int64_t)sprite_ref[0][0] * (1 << (alpha + beta + rho - min_ab))) + + ((int64_t)-r * sprite_ref[0][0] + virtual_ref[0][0]) * h3 * (-vop_ref[0][0]) + + ((int64_t)-r * sprite_ref[0][0] + virtual_ref[1][0]) * w3 * (-vop_ref[0][1]) + + ((int64_t)1 << (alpha + beta + rho - min_ab - 1)); + sprite_offset[0][1] = ((int64_t)sprite_ref[0][1] * (1 << (alpha + beta + rho - min_ab))) + + ((int64_t)-r * sprite_ref[0][1] + virtual_ref[0][1]) * h3 * (-vop_ref[0][0]) + + ((int64_t)-r * sprite_ref[0][1] + virtual_ref[1][1]) * w3 * (-vop_ref[0][1]) + + ((int64_t)1 << (alpha + beta + rho - min_ab - 1)); + sprite_offset[1][0] = ((int64_t)-r * sprite_ref[0][0] + virtual_ref[0][0]) * h3 * (-2 * vop_ref[0][0] + 1) + + ((int64_t)-r * sprite_ref[0][0] + virtual_ref[1][0]) * w3 * (-2 * vop_ref[0][1] + 1) + + (int64_t)2 * w2 * h3 * r * sprite_ref[0][0] - 16 * w2 * h3 + + ((int64_t)1 << (alpha + beta + rho - min_ab + 1)); + sprite_offset[1][1] = ((int64_t)-r * sprite_ref[0][1] + virtual_ref[0][1]) * h3 * (-2 * vop_ref[0][0] + 1) + + ((int64_t)-r * sprite_ref[0][1] + virtual_ref[1][1]) * w3 * (-2 * vop_ref[0][1] + 1) + + (int64_t)2 * w2 * h3 * r * sprite_ref[0][1] - 16 * w2 * h3 + + ((int64_t)1 << (alpha + beta + rho - min_ab + 1)); s->sprite_delta[0][0] = (-r * sprite_ref[0][0] + virtual_ref[0][0]) * h3; s->sprite_delta[0][1] = (-r * sprite_ref[0][0] + virtual_ref[1][0]) * w3; s->sprite_delta[1][0] = (-r * sprite_ref[0][1] + virtual_ref[0][1]) * h3; @@ -353,10 +346,10 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g s->sprite_delta[0][1] == 0 && s->sprite_delta[1][0] == 0 && s->sprite_delta[1][1] == a << ctx->sprite_shift[0]) { - s->sprite_offset[0][0] >>= ctx->sprite_shift[0]; - s->sprite_offset[0][1] >>= ctx->sprite_shift[0]; - s->sprite_offset[1][0] >>= ctx->sprite_shift[1]; - s->sprite_offset[1][1] >>= ctx->sprite_shift[1]; + sprite_offset[0][0] >>= ctx->sprite_shift[0]; + sprite_offset[0][1] >>= ctx->sprite_shift[0]; + sprite_offset[1][0] >>= ctx->sprite_shift[1]; + sprite_offset[1][1] >>= ctx->sprite_shift[1]; s->sprite_delta[0][0] = a; s->sprite_delta[0][1] = 0; s->sprite_delta[1][0] = 0; @@ -369,18 +362,18 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g int shift_c = 16 - ctx->sprite_shift[1]; if (shift_c < 0 || shift_y < 0 || - FFABS(s->sprite_offset[0][0]) >= INT_MAX >> shift_y || - FFABS(s->sprite_offset[1][0]) >= INT_MAX >> shift_c || - FFABS(s->sprite_offset[0][1]) >= INT_MAX >> shift_y || - FFABS(s->sprite_offset[1][1]) >= INT_MAX >> shift_c + FFABS(sprite_offset[0][0]) >= INT_MAX >> shift_y || + FFABS(sprite_offset[1][0]) >= INT_MAX >> shift_c || + FFABS(sprite_offset[0][1]) >= INT_MAX >> shift_y || + FFABS(sprite_offset[1][1]) >= INT_MAX >> shift_c ) { avpriv_request_sample(s->avctx, "Too large sprite shift or offset"); goto overflow; } for (i = 0; i < 2; i++) { - s->sprite_offset[0][i] *= 1 << shift_y; - s->sprite_offset[1][i] *= 1 << shift_c; + sprite_offset[0][i] *= 1 << shift_y; + sprite_offset[1][i] *= 1 << shift_c; s->sprite_delta[0][i] *= 1 << shift_y; s->sprite_delta[1][i] *= 1 << shift_y; ctx->sprite_shift[i] = 16; @@ -392,16 +385,16 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g s->sprite_delta[i][1] - a * (1LL<<16) }; - if (llabs(s->sprite_offset[0][i] + s->sprite_delta[i][0] * (w+16LL)) >= INT_MAX || - llabs(s->sprite_offset[0][i] + s->sprite_delta[i][1] * (h+16LL)) >= INT_MAX || - llabs(s->sprite_offset[0][i] + s->sprite_delta[i][0] * (w+16LL) + s->sprite_delta[i][1] * (h+16LL)) >= INT_MAX || + if (llabs(sprite_offset[0][i] + s->sprite_delta[i][0] * (w+16LL)) >= INT_MAX || + llabs(sprite_offset[0][i] + s->sprite_delta[i][1] * (h+16LL)) >= INT_MAX || + llabs(sprite_offset[0][i] + s->sprite_delta[i][0] * (w+16LL) + s->sprite_delta[i][1] * (h+16LL)) >= INT_MAX || llabs(s->sprite_delta[i][0] * (w+16LL)) >= INT_MAX || llabs(s->sprite_delta[i][1] * (w+16LL)) >= INT_MAX || llabs(sd[0]) >= INT_MAX || llabs(sd[1]) >= INT_MAX || - llabs(s->sprite_offset[0][i] + sd[0] * (w+16LL)) >= INT_MAX || - llabs(s->sprite_offset[0][i] + sd[1] * (h+16LL)) >= INT_MAX || - llabs(s->sprite_offset[0][i] + sd[0] * (w+16LL) + sd[1] * (h+16LL)) >= INT_MAX + llabs(sprite_offset[0][i] + sd[0] * (w+16LL)) >= INT_MAX || + llabs(sprite_offset[0][i] + sd[1] * (h+16LL)) >= INT_MAX || + llabs(sprite_offset[0][i] + sd[0] * (w+16LL) + sd[1] * (h+16LL)) >= INT_MAX ) { avpriv_request_sample(s->avctx, "Overflow on sprite points"); goto overflow; @@ -410,6 +403,11 @@ static int mpeg4_decode_sprite_trajectory(Mpeg4DecContext *ctx, GetBitContext *g s->real_sprite_warping_points = ctx->num_sprite_warping_points; } + s->sprite_offset[0][0] = sprite_offset[0][0]; + s->sprite_offset[0][1] = sprite_offset[0][1]; + s->sprite_offset[1][0] = sprite_offset[1][0]; + s->sprite_offset[1][1] = sprite_offset[1][1]; + return 0; overflow: memset(s->sprite_offset, 0, sizeof(s->sprite_offset));