From patchwork Thu Aug 13 22:06:25 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Yongle Lin X-Patchwork-Id: 21644 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 6CD8D44AACA for ; Fri, 14 Aug 2020 01:35:30 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 45A1368AD1A; Fri, 14 Aug 2020 01:35:30 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mail-qv1-f66.google.com (mail-qv1-f66.google.com [209.85.219.66]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 3D1D3680AD6 for ; Fri, 14 Aug 2020 01:35:24 +0300 (EEST) Received: by mail-qv1-f66.google.com with SMTP id b2so3436846qvp.9 for ; Thu, 13 Aug 2020 15:35:24 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=S1CGME+XSX4H3fTsdpW9oA1MqHOQdk9hi9Ts4s/FbiU=; b=CRXeHX6stQIVGv8c/8HzyDsH0b1tp95ZzoI7XFuO+M4etGJetTpHZMcH553TG+Nq+9 wJQ/viwRfrCTWG5Kh3jtRh75kGRgOboKDgHTy0htsBRyuVjCdfl7zXlQaO/Axx6VYmjA 23Cdu5Eds7kO2yFM6fRq1nQa5E+f4TSi2ZuJmt0WC0LkX5TEoSXk/7D5JaKo9XsWNsC/ Ar2D3ojsB4vws248L+y9GkSBs4TuKaBCroRFOJSPWhv1r9yreiIjoYLdXB2glMbKvYqn KvIA0IMzwnZbIT1JAhZA9inWGosOSUqe8LFi0/EwAWTLx53rZf4UGZwJpi4qXviV3I4+ hXjg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=S1CGME+XSX4H3fTsdpW9oA1MqHOQdk9hi9Ts4s/FbiU=; b=GqtjAbUOVx+ohGJidTLulkf2r4O1mLtsAsJcIHzrRkzjYEAyLDdAblUQVwuse0k7oP PCqM3FQmZxEpXJtGX0U475KbMCP2G3YzE0PMLQoCQty4v2oFdmLseWwwa4caKkbFotwu ykC+9jfoTxY7iGABCdCdUz3b/K8/vannxh1Ikbdk654MHhf+NFKy78iqQHMmrPDzopGd X9XiWzct/LwqXK9O+19wtkROTvWes/9hsivkN9rpNRJodj++xiKh7wANpsjCpdZhwodU dp3MIqYpjrUt/4fAdDILl4yP+d/qO4vU7HnyOnjfDas5NRBo8fkHGXqzv012l4762oF3 zO5g== X-Gm-Message-State: AOAM531tpmV12IHBz8DpHU5yM3YspQtiTdLMOO2r8RiX3fckrt8oB7pW cfcvp8Gd+I49W+/dbn3ZCIW636My X-Google-Smtp-Source: ABdhPJwvJFPM+N4G0UCUVXz98YmpyyFvwof5jWaNS/r7FTlT73/dEZAZ7venJzots2TLY/QqMwjTnQ== X-Received: by 2002:a17:902:6ac9:: with SMTP id i9mr5692296plt.128.1597356431083; Thu, 13 Aug 2020 15:07:11 -0700 (PDT) Received: from yonglel.c.googlers.com.com (219.29.83.34.bc.googleusercontent.com. [34.83.29.219]) by smtp.gmail.com with ESMTPSA id s22sm6895699pfh.16.2020.08.13.15.07.10 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 13 Aug 2020 15:07:10 -0700 (PDT) From: Yongle Lin X-Google-Original-From: Yongle Lin To: ffmpeg-devel@ffmpeg.org Date: Thu, 13 Aug 2020 22:06:25 +0000 Message-Id: <20200813220625.292120-1-yonglel@google.com> X-Mailer: git-send-email 2.28.0.220.ged08abb693-goog MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH] libavcodec/vp9: export motion vector to side data in vp9 X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Yongle Lin Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" --- libavcodec/vp9.c | 102 ++++++++++++++++++++++++++++++++++++++++++ libavcodec/vp9block.c | 2 + libavcodec/vp9dec.h | 3 ++ 3 files changed, 107 insertions(+) diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c index fd0bab14a2..5289fb3099 100644 --- a/libavcodec/vp9.c +++ b/libavcodec/vp9.c @@ -35,6 +35,7 @@ #include "libavutil/avassert.h" #include "libavutil/pixdesc.h" #include "libavutil/video_enc_params.h" +#include "libavutil/motion_vector.h" #define VP9_SYNCCODE 0x498342 @@ -99,6 +100,7 @@ static void vp9_tile_data_free(VP9TileData *td) av_freep(&td->b_base); av_freep(&td->block_base); av_freep(&td->block_structure); + av_freep(&td->block_motion_vectors); } static void vp9_frame_unref(AVCodecContext *avctx, VP9Frame *f) @@ -334,6 +336,11 @@ static int update_block_buffers(AVCodecContext *avctx) if (!td->block_structure) return AVERROR(ENOMEM); } + if (avctx->export_side_data & AV_CODEC_EXPORT_DATA_MVS) { + td->block_motion_vectors = av_malloc_array(s->cols * s->rows, sizeof(*td->block_motion_vectors)); + if (!td->block_motion_vectors) + return AVERROR(ENOMEM); + } } else { for (i = 1; i < s->active_tile_cols; i++) vp9_tile_data_free(&s->td[i]); @@ -355,6 +362,11 @@ static int update_block_buffers(AVCodecContext *avctx) if (!s->td[i].block_structure) return AVERROR(ENOMEM); } + if (avctx->export_side_data & AV_CODEC_EXPORT_DATA_MVS) { + td->block_motion_vectors = av_malloc_array(s->cols * s->rows, sizeof(*td->block_motion_vectors)); + if (!td->block_motion_vectors) + return AVERROR(ENOMEM); + } } } s->block_alloc_using_2pass = s->s.frames[CUR_FRAME].uses_2pass; @@ -1548,6 +1560,92 @@ static int vp9_export_enc_params(VP9Context *s, VP9Frame *frame) return 0; } +static int add_mv(AVMotionVector *mb, int w, int h, + int dst_x, int dst_y, + int motion_x, int motion_y, int motion_scale, + int direction) +{ + mb->w = w; + mb->h = h; + mb->motion_x = motion_x; + mb->motion_y = motion_y; + mb->motion_scale = motion_scale; + mb->dst_x = dst_x; + mb->dst_y = dst_y; + mb->src_x = dst_x + motion_x / motion_scale; + mb->src_y = dst_y + motion_y / motion_scale; + mb->source = direction ? 1 : -1; + mb->flags = 0; // XXX: does mb_type contain extra information that could be exported here? + return 1; +} + +static int vp9_export_mv(AVCodecContext *avctx, AVFrame *frame) +{ + VP9Context *s = avctx->priv_data; + AVMotionVector *mvs = av_malloc_array(frame->width * frame->height, 2 * 4 * sizeof(AVMotionVector)); + if (!mvs) + return AVERROR(ENOMEM); + + unsigned int tile, nb_blocks = 0; + for (tile = 0; tile < s->active_tile_cols; tile++) { + nb_blocks += s->td[tile].nb_block_structure; + } + + if (nb_blocks) { + unsigned int block = 0; + unsigned int tile, block_tile; + + for (tile = 0; tile < s->active_tile_cols; tile++) { + VP9TileData *td = &s->td[tile]; + + for (block_tile = 0; block_tile < td->nb_block_structure; block_tile++) { + unsigned int row = td->block_structure[block_tile].row; + unsigned int col = td->block_structure[block_tile].col; + int w = 1 << (3 + td->block_structure[block_tile].block_size_idx_x); + int h = 1 << (3 + td->block_structure[block_tile].block_size_idx_y); + int src_x = col * 8; + int src_y = row * 8; + int motion_x, motion_y, dst_x, dst_y; + if (w >= 8 && h >= 8) { + for (int ref = 0; ref < 2; ref++) { + motion_x = td->block_motion_vectors[block_tile].mv[0][ref].x; + motion_y = td->block_motion_vectors[block_tile].mv[0][ref].y; + dst_x = src_x + w / 2; + dst_y = src_y + w / 2; + add_mv(mvs + block, w, h, dst_x, dst_y, motion_x, motion_y, 16, ref); + block++; + } + } else { + for (int b_idx = 0; b_idx < 4; b_idx++) { + for (int ref = 0; ref < 2; ref++) { + motion_x = td->block_motion_vectors[block_tile].mv[b_idx][ref].x; + motion_y = td->block_motion_vectors[block_tile].mv[b_idx][ref].y; + dst_x = src_x + (b_idx % 2 + 1) * w / 4; + dst_y = src_y + (b_idx / 2 + 1) * w / 4; + add_mv(mvs + block, w, h, dst_x, dst_y, motion_x, motion_y, 16, ref); + block++; + } + } + } + } + } + + if (block) { + AVFrameSideData *sd; + + av_log(avctx, AV_LOG_DEBUG, "Adding %d MVs info to frame %d\n", block, avctx->frame_number); + sd = av_frame_new_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS, block * sizeof(AVMotionVector)); + if (!sd) { + av_freep(&mvs); + return -1; + } + memcpy(sd->data, mvs, block * sizeof(AVMotionVector)); + } + av_freep(&mvs); + } + return 0; +} + static int vp9_decode_frame(AVCodecContext *avctx, void *frame, int *got_frame, AVPacket *pkt) { @@ -1778,6 +1876,10 @@ finish: *got_frame = 1; } + if (avctx->export_side_data & AV_CODEC_EXPORT_DATA_MVS) + if (ret = vp9_export_mv(avctx, frame) < 0) + return ret; + return pkt->size; } diff --git a/libavcodec/vp9block.c b/libavcodec/vp9block.c index ec16e26c69..2bf1d157cf 100644 --- a/libavcodec/vp9block.c +++ b/libavcodec/vp9block.c @@ -1297,6 +1297,8 @@ void ff_vp9_decode_block(VP9TileData *td, int row, int col, td->block_structure[td->nb_block_structure].block_size_idx_y = av_log2(h4); td->nb_block_structure++; } + if (td->block_motion_vectors) + memcpy(&td->block_motion_vectors[td->nb_block_structure].mv, &b->mv, sizeof(b->mv)); if (!b->skip) { int has_coeffs; diff --git a/libavcodec/vp9dec.h b/libavcodec/vp9dec.h index cc2440b854..a05ce172ed 100644 --- a/libavcodec/vp9dec.h +++ b/libavcodec/vp9dec.h @@ -232,6 +232,9 @@ struct VP9TileData { unsigned int block_size_idx_x:2; unsigned int block_size_idx_y:2; } *block_structure; + struct { + VP56mv mv[4][2]; + } *block_motion_vectors; unsigned int nb_block_structure; };