[FFmpeg-devel,V4] avcodec/libvpxenc: add VP8 support for ROI-based encoding

Submitted by Guo, Yejun on March 8, 2019, 8:57 p.m.

Details

Message ID 1552078632-16096-1-git-send-email-yejun.guo@intel.com
State New
Headers show

Commit Message

Guo, Yejun March 8, 2019, 8:57 p.m.
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
---
 libavcodec/libvpxenc.c | 150 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)

Comments

James Zern March 12, 2019, 11:44 p.m.
On Fri, Mar 8, 2019 at 5:07 AM Guo, Yejun <yejun.guo@intel.com> wrote:
>
> Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
> ---
>  libavcodec/libvpxenc.c | 150 +++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 150 insertions(+)
>
> [...]
> +    active_map.active_map = av_malloc(active_map.rows * active_map.cols);
> +    if (!active_map.active_map) {
> +        av_log(avctx, AV_LOG_ERROR, "active_map alloc failed.\n");
> +        ret = AVERROR(ENOMEM);
> +        goto fail;
> +    }
> +    /* set 1 to enable the corresponding element of vpx_roi_map_t.roi_map. */
> +    memset(active_map.active_map, 1, active_map.rows * active_map.cols);
>

ROI is independent of active map, you don't need it for ROI to work.
In this case you're setting the entire frame as active which may not
be what you want. For this patch maybe we should focus on only setting
the ROI.
Guo, Yejun March 13, 2019, 7:27 a.m.
> -----Original Message-----

> From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf

> Of James Zern

> Sent: Wednesday, March 13, 2019 7:45 AM

> To: FFmpeg development discussions and patches <ffmpeg-

> devel@ffmpeg.org>

> Subject: Re: [FFmpeg-devel] [PATCH V4] avcodec/libvpxenc: add VP8

> support for ROI-based encoding

> 

> On Fri, Mar 8, 2019 at 5:07 AM Guo, Yejun <yejun.guo@intel.com> wrote:

> >

> > Signed-off-by: Guo, Yejun <yejun.guo@intel.com>

> > ---

> >  libavcodec/libvpxenc.c | 150

> +++++++++++++++++++++++++++++++++++++++++++++++++

> >  1 file changed, 150 insertions(+)

> >

> > [...]

> > +    active_map.active_map = av_malloc(active_map.rows *

> active_map.cols);

> > +    if (!active_map.active_map) {

> > +        av_log(avctx, AV_LOG_ERROR, "active_map alloc failed.\n");

> > +        ret = AVERROR(ENOMEM);

> > +        goto fail;

> > +    }

> > +    /* set 1 to enable the corresponding element of

> vpx_roi_map_t.roi_map. */

> > +    memset(active_map.active_map, 1, active_map.rows *

> active_map.cols);

> >

> 

> ROI is independent of active map, you don't need it for ROI to work.

> In this case you're setting the entire frame as active which may not

> be what you want. For this patch maybe we should focus on only setting

> the ROI.


yes, I mis-understood the libvpx API, will send out a new version after Mark T's patches pushed.

> _______________________________________________

> ffmpeg-devel mailing list

> ffmpeg-devel@ffmpeg.org

> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

Patch hide | download patch | download mbox

diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
index c823b8a..2984c7a 100644
--- a/libavcodec/libvpxenc.c
+++ b/libavcodec/libvpxenc.c
@@ -1057,6 +1057,153 @@  static int queue_frames(AVCodecContext *avctx, AVPacket *pkt_out)
     return size;
 }
 
+static int vp8_encode_set_roi(AVCodecContext *avctx, const AVFrame *frame)
+{
+    /* range of vpx_roi_map_t.delta_q[i] is [-63, 63] */
+#define MAX_DELTA_Q 63
+
+    const AVRegionOfInterest *roi = NULL;
+    vpx_roi_map_t roi_map;
+    int ret = 0;
+    int nb_rois;
+    AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
+    VPxContext *ctx = avctx->priv_data;
+    vpx_active_map_t active_map = { 0 };
+    int max_segment_cnt = 4;    /* VP8 ROI only support 4 segments. */
+    int segment_id = 0;
+
+    /* record the mapping from delta_q to "segment id + 1".
+     * delta_q is shift with MAX_DELTA_Q, and so the range is [0, 2*MAX_DELTA_Q].
+     * add 1 to segment id, so no mapping if the value of array element is zero.
+     */
+    int segment_mapping[2 * MAX_DELTA_Q + 1] = { 0 };
+
+    /* segment id 0 in roi_map is reserved for the areas not covered by AVRegionOfInterest.
+     * segment id 0 in roi_map is also for the areas with AVRegionOfInterest.qoffset near 0.
+     */
+    segment_mapping[MAX_DELTA_Q] = segment_id + 1;
+    segment_id++;
+
+    active_map.rows = (frame->height + 15) / 16;
+    active_map.cols = (frame->width  + 15) / 16;
+
+    if (!sd) {
+        /* For the case that some middle frames do not have ROI while other frames have ROIs.
+         * Due to libvpx behavior, we have to reset VP8E_SET_ACTIVEMAP, otherwise the previous
+         * ROIs continue working for frames without ROIs.
+         */
+        if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP, &active_map)) {
+            log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec control.\n");
+            return AVERROR_INVALIDDATA;
+        }
+        return 0;
+    }
+
+    memset(&roi_map, 0, sizeof(roi_map));
+    roi_map.rows = active_map.rows;
+    roi_map.cols = active_map.cols;
+    roi_map.roi_map = av_mallocz(roi_map.rows * roi_map.cols);
+    if (!roi_map.roi_map) {
+        av_log(avctx, AV_LOG_ERROR, "roi_map alloc failed.\n");
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    active_map.active_map = av_malloc(active_map.rows * active_map.cols);
+    if (!active_map.active_map) {
+        av_log(avctx, AV_LOG_ERROR, "active_map alloc failed.\n");
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+    /* set 1 to enable the corresponding element of vpx_roi_map_t.roi_map. */
+    memset(active_map.active_map, 1, active_map.rows * active_map.cols);
+
+    roi = (const AVRegionOfInterest*)sd->data;
+    if (!roi->self_size || sd->size % roi->self_size != 0) {
+        av_log(ctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+    nb_rois = sd->size / roi->self_size;
+
+    /* This list must be iterated from zero because regions are
+     * defined in order of decreasing importance. So discard less
+     * important areas if they exceed the segment count.
+     */
+    for (int i = 0; i < nb_rois; i++) {
+        int qoffset;
+        int mapping_index;
+
+        roi = (const AVRegionOfInterest*)(sd->data + roi->self_size * i);
+        if (roi->qoffset.den == 0) {
+            av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n");
+            ret = AVERROR(EINVAL);
+            goto fail;
+        }
+
+        qoffset = (int)(roi->qoffset.num * 1.0f / roi->qoffset.den * MAX_DELTA_Q);
+        qoffset = av_clip(qoffset, -MAX_DELTA_Q, MAX_DELTA_Q);
+
+        mapping_index = qoffset + MAX_DELTA_Q;
+        if (!segment_mapping[mapping_index]) {
+            if (segment_id > max_segment_cnt - 1) {
+                av_log(ctx, AV_LOG_WARNING,
+                       "ROI only supports %d segments (and segment 0 is reserved for non-ROIs), skipping this one.\n",
+                       max_segment_cnt);
+                roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);
+                continue;
+            }
+
+            segment_mapping[mapping_index] = segment_id + 1;
+            roi_map.delta_q[segment_id] = qoffset;
+            segment_id++;
+        }
+    }
+
+    /* This list must be iterated in reverse, so for the case that
+     * two regions overlapping, the more important area takes effect.
+     */
+    for (int i = nb_rois - 1; i >= 0; i--) {
+        int qoffset;
+        int mapping_value;
+        int starty, endy, startx, endx;
+
+        roi = (const AVRegionOfInterest*)(sd->data + roi->self_size * i);
+
+        starty = FFMIN(roi_map.rows, roi->top / 16);
+        endy   = FFMIN(roi_map.rows, (roi->bottom + 15) / 16);
+        startx = FFMIN(roi_map.cols, roi->left / 16);
+        endx   = FFMIN(roi_map.cols, (roi->right + 15) / 16);
+
+        qoffset = (int)(roi->qoffset.num * 1.0f / roi->qoffset.den * MAX_DELTA_Q);
+        qoffset = av_clip(qoffset, -MAX_DELTA_Q, MAX_DELTA_Q);
+
+        mapping_value = segment_mapping[qoffset + MAX_DELTA_Q];
+        if (mapping_value) {
+            for (int y = starty; y < endy; y++)
+                for (int x = startx; x < endx; x++)
+                    roi_map.roi_map[x + y * roi_map.cols] = mapping_value - 1;
+        }
+    }
+
+    if (vpx_codec_control(&ctx->encoder, VP8E_SET_ROI_MAP, &roi_map)) {
+        log_encoder_error(avctx, "Failed to set VP8E_SET_ROI_MAP codec control.\n");
+        ret = AVERROR_INVALIDDATA;
+        goto fail;
+    }
+
+    if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP, &active_map)) {
+        log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec control.\n");
+        ret = AVERROR_INVALIDDATA;
+        goto fail;
+    }
+
+fail:
+    av_freep(&active_map.active_map);
+    av_freep(&roi_map.roi_map);
+    return ret;
+}
+
 static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
                       const AVFrame *frame, int *got_packet)
 {
@@ -1113,6 +1260,9 @@  static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
                 flags |= strtoul(en->value, NULL, 10);
             }
         }
+
+        if (avctx->codec_id == AV_CODEC_ID_VP8)
+            vp8_encode_set_roi(avctx, frame);
     }
 
     res = vpx_codec_encode(&ctx->encoder, rawimg, timestamp,