[FFmpeg-devel,V2] avcodec/libvpxenc: add VP8 support for ROI-based encoding

Submitted by Guo, Yejun on Feb. 28, 2019, 7:10 p.m.

Details

Message ID 1551381020-18742-1-git-send-email-yejun.guo@intel.com
State New
Headers show

Commit Message

Guo, Yejun Feb. 28, 2019, 7:10 p.m.
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
---
 libavcodec/libvpxenc.c | 131 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)

Comments

Guo, Yejun March 1, 2019, 3:23 a.m.
> -----Original Message-----

> From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf

> Of Guo, Yejun

> Sent: Friday, March 01, 2019 3:10 AM

> To: ffmpeg-devel@ffmpeg.org

> Subject: [FFmpeg-devel] [PATCH V2] avcodec/libvpxenc: add VP8 support for

> ROI-based encoding

> 

> Signed-off-by: Guo, Yejun <yejun.guo@intel.com>

> ---

>  libavcodec/libvpxenc.c | 131

> +++++++++++++++++++++++++++++++++++++++++++++++++

>  1 file changed, 131 insertions(+)

> 

> diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c

> index c823b8a..0f8efcc 100644

> --- a/libavcodec/libvpxenc.c

> +++ b/libavcodec/libvpxenc.c

> @@ -1057,6 +1057,134 @@ static int queue_frames(AVCodecContext *avctx,

> AVPacket *pkt_out)

>      return size;

>  }

> 

> +static int vp8_encode_set_roi(AVCodecContext *avctx, const AVFrame

> *frame)

> +{

> +    /* range of vpx_roi_map_t.delta_q[i] is [-63, 63] */

> +#define MAX_DELTA_Q 63

> +

> +    const AVRegionOfInterest *roi = NULL;

> +    vpx_roi_map_t roi_map;

> +    int ret = 0;

> +    int nb_rois;

> +    AVFrameSideData *sd = av_frame_get_side_data(frame,

> AV_FRAME_DATA_REGIONS_OF_INTEREST);

> +    VPxContext *ctx = avctx->priv_data;

> +    vpx_active_map_t active_map = { 0, 0, 0 };

> +    int segment_id = 0;

> +

> +    /* record the mapping from delta_q to "segment id + 1".

> +     * delta_q is shift with MAX_DELTA_Q, and so the range is [0,

> 2*MAX_DELTA_Q].

> +     * add 1 to segment id, so no mapping if the value of array element is zero.

> +     */

> +    int segment_mapping[2 * MAX_DELTA_Q + 1] = {0};

> +

> +    /* segment id 0 in roi_map is reserved for the areas not covered by

> AVRegionOfInterest.

> +     * segment id 0 in roi_map is also for the areas with

> AVRegionOfInterest.qoffset near 0.

> +     */

> +    segment_mapping[MAX_DELTA_Q] = segment_id + 1;

> +    segment_id++;

> +

> +    active_map.rows = (frame->height + 15) / 16;

> +    active_map.cols = (frame->width  + 15) / 16;

> +

> +    if (!sd) {

> +        /* For the case that some middle frames do not have ROI while other

> frames have ROIs.

> +         * Due to libvpx behaivor, we have to reset VP8E_SET_ACTIVEMAP,

> otherwise the previous

> +         * ROIs continue working for frames without ROIs.

> +         */

> +        if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP,

> &active_map)) {

> +            log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec

> control.\n");

> +            return AVERROR_INVALIDDATA;

> +        }

> +        return 0;

> +    }

> +

> +    memset(&roi_map, 0, sizeof(roi_map));

> +    roi_map.rows = active_map.rows;

> +    roi_map.cols = active_map.cols;

> +    roi_map.roi_map = av_mallocz(roi_map.rows * roi_map.cols);

> +    if (!roi_map.roi_map) {

> +        av_log(avctx, AV_LOG_ERROR, "roi_map alloc failed.\n");

> +        ret = AVERROR(ENOMEM);

> +        goto fail;

> +    }

> +

> +    active_map.active_map = av_malloc(active_map.rows * active_map.cols);

> +    if (!active_map.active_map) {

> +        av_log(avctx, AV_LOG_ERROR, "active_map alloc failed.\n");

> +        ret = AVERROR(ENOMEM);

> +        goto fail;

> +    }

> +    /* set 1 to enable the corresponding element of vpx_roi_map_t.roi_map.

> */

> +    memset(active_map.active_map, 1, active_map.rows * active_map.cols);

> +

> +    roi = (const AVRegionOfInterest*)sd->data;

> +    if (!roi->self_size || sd->size % roi->self_size != 0) {

> +        av_log(ctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");

> +        ret = AVERROR(EINVAL);

> +        goto fail;

> +    }

> +    nb_rois = sd->size / roi->self_size;

> +

> +    // This list must be iterated in reverse because regions are

> +    // defined in order of decreasing importance.

> +    for (int i = nb_rois - 1; i >= 0; i--) {

> +        int qoffset;

> +        int mapping_index;

> +        int mapping_value;

> +        int starty = FFMIN(roi_map.rows, roi->top / 16);

> +        int endy   = FFMIN(roi_map.rows, (roi->bottom + 15) / 16);

> +        int startx = FFMIN(roi_map.cols, roi->left / 16);

> +        int endx   = FFMIN(roi_map.cols, (roi->right + 15) / 16);

> +

> +        roi = (const AVRegionOfInterest*)(sd->data + roi->self_size * i);

> +        if (roi->qoffset.den == 0) {

> +            av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must

> not be zero.\n");

> +            ret = AVERROR(EINVAL);

> +            goto fail;

> +        }

> +

> +        qoffset = (int)(roi->qoffset.num * 1.0f / roi->qoffset.den *

> MAX_DELTA_Q);

> +        qoffset = av_clip(qoffset, -MAX_DELTA_Q, MAX_DELTA_Q);

> +

> +        mapping_index = qoffset + MAX_DELTA_Q;

> +        if (!segment_mapping[mapping_index]) {

> +            if (segment_id > 3) {

> +                av_log(ctx, AV_LOG_WARNING,

> +                       "ROI only support 4 segments (and segment 0 is reserved for

> non-ROIs), skipping this one.\n");

> +                roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);

> +                continue;


please ignore this patch v2, because the most important ROI is ignored if this case happens.
I'll send out v3.

> +            }

> +

> +            segment_mapping[mapping_index] = segment_id + 1;

> +            roi_map.delta_q[segment_id] = qoffset;

> +            segment_id++;

> +        }

> +

> +        mapping_value = segment_mapping[mapping_index];

> +

> +        for (int y = starty; y < endy; y++)

> +            for (int x = startx; x < endx; x++)

> +                roi_map.roi_map[x + y * roi_map.cols] = mapping_value - 1;

> +    }

> +

> +    if (vpx_codec_control(&ctx->encoder, VP8E_SET_ROI_MAP, &roi_map))

> {

> +        log_encoder_error(avctx, "Failed to set VP8E_SET_ROI_MAP codec

> control.\n");

> +        ret = AVERROR_INVALIDDATA;

> +        goto fail;

> +    }

> +

> +    if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP,

> &active_map)) {

> +        log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec

> control.\n");

> +        ret = AVERROR_INVALIDDATA;

> +        goto fail;

> +    }

> +

> +fail:

> +    av_freep(&active_map.active_map);

> +    av_freep(&roi_map.roi_map);

> +    return ret;

> +}

> +

>  static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,

>                        const AVFrame *frame, int *got_packet)

>  {

> @@ -1113,6 +1241,9 @@ static int vpx_encode(AVCodecContext *avctx,

> AVPacket *pkt,

>                  flags |= strtoul(en->value, NULL, 10);

>              }

>          }

> +

> +        if (avctx->codec_id == AV_CODEC_ID_VP8)

> +            vp8_encode_set_roi(avctx, frame);

>      }

> 

>      res = vpx_codec_encode(&ctx->encoder, rawimg, timestamp,

> --

> 2.7.4

> 

> _______________________________________________

> ffmpeg-devel mailing list

> ffmpeg-devel@ffmpeg.org

> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

Patch hide | download patch | download mbox

diff --git a/libavcodec/libvpxenc.c b/libavcodec/libvpxenc.c
index c823b8a..0f8efcc 100644
--- a/libavcodec/libvpxenc.c
+++ b/libavcodec/libvpxenc.c
@@ -1057,6 +1057,134 @@  static int queue_frames(AVCodecContext *avctx, AVPacket *pkt_out)
     return size;
 }
 
+static int vp8_encode_set_roi(AVCodecContext *avctx, const AVFrame *frame)
+{
+    /* range of vpx_roi_map_t.delta_q[i] is [-63, 63] */
+#define MAX_DELTA_Q 63
+
+    const AVRegionOfInterest *roi = NULL;
+    vpx_roi_map_t roi_map;
+    int ret = 0;
+    int nb_rois;
+    AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
+    VPxContext *ctx = avctx->priv_data;
+    vpx_active_map_t active_map = { 0, 0, 0 };
+    int segment_id = 0;
+
+    /* record the mapping from delta_q to "segment id + 1".
+     * delta_q is shift with MAX_DELTA_Q, and so the range is [0, 2*MAX_DELTA_Q].
+     * add 1 to segment id, so no mapping if the value of array element is zero.
+     */
+    int segment_mapping[2 * MAX_DELTA_Q + 1] = {0};
+
+    /* segment id 0 in roi_map is reserved for the areas not covered by AVRegionOfInterest.
+     * segment id 0 in roi_map is also for the areas with AVRegionOfInterest.qoffset near 0.
+     */
+    segment_mapping[MAX_DELTA_Q] = segment_id + 1;
+    segment_id++;
+
+    active_map.rows = (frame->height + 15) / 16;
+    active_map.cols = (frame->width  + 15) / 16;
+
+    if (!sd) {
+        /* For the case that some middle frames do not have ROI while other frames have ROIs.
+         * Due to libvpx behaivor, we have to reset VP8E_SET_ACTIVEMAP, otherwise the previous
+         * ROIs continue working for frames without ROIs.
+         */
+        if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP, &active_map)) {
+            log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec control.\n");
+            return AVERROR_INVALIDDATA;
+        }
+        return 0;
+    }
+
+    memset(&roi_map, 0, sizeof(roi_map));
+    roi_map.rows = active_map.rows;
+    roi_map.cols = active_map.cols;
+    roi_map.roi_map = av_mallocz(roi_map.rows * roi_map.cols);
+    if (!roi_map.roi_map) {
+        av_log(avctx, AV_LOG_ERROR, "roi_map alloc failed.\n");
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    active_map.active_map = av_malloc(active_map.rows * active_map.cols);
+    if (!active_map.active_map) {
+        av_log(avctx, AV_LOG_ERROR, "active_map alloc failed.\n");
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+    /* set 1 to enable the corresponding element of vpx_roi_map_t.roi_map. */
+    memset(active_map.active_map, 1, active_map.rows * active_map.cols);
+
+    roi = (const AVRegionOfInterest*)sd->data;
+    if (!roi->self_size || sd->size % roi->self_size != 0) {
+        av_log(ctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+    nb_rois = sd->size / roi->self_size;
+
+    // This list must be iterated in reverse because regions are
+    // defined in order of decreasing importance.
+    for (int i = nb_rois - 1; i >= 0; i--) {
+        int qoffset;
+        int mapping_index;
+        int mapping_value;
+        int starty = FFMIN(roi_map.rows, roi->top / 16);
+        int endy   = FFMIN(roi_map.rows, (roi->bottom + 15) / 16);
+        int startx = FFMIN(roi_map.cols, roi->left / 16);
+        int endx   = FFMIN(roi_map.cols, (roi->right + 15) / 16);
+
+        roi = (const AVRegionOfInterest*)(sd->data + roi->self_size * i);
+        if (roi->qoffset.den == 0) {
+            av_log(ctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n");
+            ret = AVERROR(EINVAL);
+            goto fail;
+        }
+
+        qoffset = (int)(roi->qoffset.num * 1.0f / roi->qoffset.den * MAX_DELTA_Q);
+        qoffset = av_clip(qoffset, -MAX_DELTA_Q, MAX_DELTA_Q);
+
+        mapping_index = qoffset + MAX_DELTA_Q;
+        if (!segment_mapping[mapping_index]) {
+            if (segment_id > 3) {
+                av_log(ctx, AV_LOG_WARNING,
+                       "ROI only support 4 segments (and segment 0 is reserved for non-ROIs), skipping this one.\n");
+                roi = (AVRegionOfInterest*)((char*)roi + roi->self_size);
+                continue;
+            }
+
+            segment_mapping[mapping_index] = segment_id + 1;
+            roi_map.delta_q[segment_id] = qoffset;
+            segment_id++;
+        }
+
+        mapping_value = segment_mapping[mapping_index];
+
+        for (int y = starty; y < endy; y++)
+            for (int x = startx; x < endx; x++)
+                roi_map.roi_map[x + y * roi_map.cols] = mapping_value - 1;
+    }
+
+    if (vpx_codec_control(&ctx->encoder, VP8E_SET_ROI_MAP, &roi_map)) {
+        log_encoder_error(avctx, "Failed to set VP8E_SET_ROI_MAP codec control.\n");
+        ret = AVERROR_INVALIDDATA;
+        goto fail;
+    }
+
+    if (vpx_codec_control(&ctx->encoder, VP8E_SET_ACTIVEMAP, &active_map)) {
+        log_encoder_error(avctx, "Failed to set VP8E_SET_ACTIVEMAP codec control.\n");
+        ret = AVERROR_INVALIDDATA;
+        goto fail;
+    }
+
+fail:
+    av_freep(&active_map.active_map);
+    av_freep(&roi_map.roi_map);
+    return ret;
+}
+
 static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
                       const AVFrame *frame, int *got_packet)
 {
@@ -1113,6 +1241,9 @@  static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
                 flags |= strtoul(en->value, NULL, 10);
             }
         }
+
+        if (avctx->codec_id == AV_CODEC_ID_VP8)
+            vp8_encode_set_roi(avctx, frame);
     }
 
     res = vpx_codec_encode(&ctx->encoder, rawimg, timestamp,