diff mbox series

[FFmpeg-devel] avfilter/vf_overlay_cuda: support expression of x y position

Message ID 20210608062053.16308-1-liuqi05@kuaishou.com
State Superseded
Headers show
Series [FFmpeg-devel] avfilter/vf_overlay_cuda: support expression of x y position
Related show

Checks

Context Check Description
andriy/x86_make success Make finished
andriy/x86_make_fate success Make fate finished
andriy/PPC64_make success Make finished
andriy/PPC64_make_fate success Make fate finished

Commit Message

Steven Liu June 8, 2021, 6:20 a.m. UTC
Signed-off-by: Steven Liu <liuqi05@kuaishou.com>
---
 libavfilter/vf_overlay_cuda.c | 129 +++++++++++++++++++++++++++++++---
 1 file changed, 120 insertions(+), 9 deletions(-)

Comments

Timo Rothenpieler June 8, 2021, 10:23 a.m. UTC | #1
It'd probably make sense to also copy over the option for evaluating the 
expression per-frame or on init.
Given that this is usually going to run much faster than the software 
overlay filter, the slight overhead from the expression could be much 
more severe.

Looks good otherwise, and for what I care, the per-frame/init option can 
also be added in a later patch.
Steven Liu June 8, 2021, 10:50 a.m. UTC | #2
Timo Rothenpieler <timo@rothenpieler.org> 于2021年6月8日周二 下午6:23写道:
>
> It'd probably make sense to also copy over the option for evaluating the
> expression per-frame or on init.
> Given that this is usually going to run much faster than the software
> overlay filter, the slight overhead from the expression could be much
> more severe.

Thanks for your comments Timo,

I update the patch version 2 right now.
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20210608104503.25677-1-liuqi05@kuaishou.com/

>
> Looks good otherwise, and for what I care, the per-frame/init option can
> also be added in a later patch.
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Thanks
Steven
diff mbox series

Patch

diff --git a/libavfilter/vf_overlay_cuda.c b/libavfilter/vf_overlay_cuda.c
index 8a4d2c4312..9980b0137e 100644
--- a/libavfilter/vf_overlay_cuda.c
+++ b/libavfilter/vf_overlay_cuda.c
@@ -30,6 +30,7 @@ 
 #include "libavutil/hwcontext.h"
 #include "libavutil/hwcontext_cuda_internal.h"
 #include "libavutil/cuda_check.h"
+#include "libavutil/eval.h"
 
 #include "avfilter.h"
 #include "framesync.h"
@@ -41,6 +42,9 @@ 
 #define BLOCK_X 32
 #define BLOCK_Y 16
 
+#define MAIN    0
+#define OVERLAY 1
+
 static const enum AVPixelFormat supported_main_formats[] = {
     AV_PIX_FMT_NV12,
     AV_PIX_FMT_YUV420P,
@@ -54,6 +58,32 @@  static const enum AVPixelFormat supported_overlay_formats[] = {
     AV_PIX_FMT_NONE,
 };
 
+enum var_name {
+    VAR_MAIN_W,    VAR_MW,
+    VAR_MAIN_H,    VAR_MH,
+    VAR_OVERLAY_W, VAR_OW,
+    VAR_OVERLAY_H, VAR_OH,
+    VAR_X,
+    VAR_Y,
+    VAR_N,
+    VAR_POS,
+    VAR_T,
+    VAR_VARS_NB
+};
+
+static const char *const var_names[] = {
+    "main_w",    "W", ///< width  of the main    video
+    "main_h",    "H", ///< height of the main    video
+    "overlay_w", "w", ///< width  of the overlay video
+    "overlay_h", "h", ///< height of the overlay video
+    "x",
+    "y",
+    "n",            ///< number of frame
+    "pos",          ///< position in the file
+    "t",            ///< timestamp expressed in seconds
+    NULL
+};
+
 /**
  * OverlayCUDAContext
  */
@@ -76,6 +106,10 @@  typedef struct OverlayCUDAContext {
     int x_position;
     int y_position;
 
+    double var_values[VAR_VARS_NB];
+    char *x_expr, *y_expr;
+
+    AVExpr *x_pexpr, *y_pexpr;
 } OverlayCUDAContext;
 
 /**
@@ -89,6 +123,49 @@  static int format_is_supported(const enum AVPixelFormat formats[], enum AVPixelF
     return 0;
 }
 
+static inline int normalize_xy(double d, int chroma_sub)
+{
+    if (isnan(d))
+        return INT_MAX;
+    return (int)d & ~((1 << chroma_sub) - 1);
+}
+
+static void eval_expr(AVFilterContext *ctx)
+{
+    OverlayCUDAContext *s = ctx->priv;
+
+    s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values, NULL);
+    s->var_values[VAR_Y] = av_expr_eval(s->y_pexpr, s->var_values, NULL);
+    /* It is necessary if x is expressed from y  */
+    s->var_values[VAR_X] = av_expr_eval(s->x_pexpr, s->var_values, NULL);
+
+    s->x_position = normalize_xy(s->var_values[VAR_X], 1);
+
+    /* the cuda pixel format is using hwaccel, y_position unnecessary normalize y */
+    s->y_position = s->var_values[VAR_Y];
+}
+
+static int set_expr(AVExpr **pexpr, const char *expr, const char *option, void *log_ctx)
+{
+    int ret;
+    AVExpr *old = NULL;
+
+    if (*pexpr)
+        old = *pexpr;
+    ret = av_expr_parse(pexpr, expr, var_names,
+                        NULL, NULL, NULL, NULL, 0, log_ctx);
+    if (ret < 0) {
+        av_log(log_ctx, AV_LOG_ERROR,
+               "Error when evaluating the expression '%s' for %s\n",
+               expr, option);
+        *pexpr = old;
+        return ret;
+    }
+
+    av_expr_free(old);
+    return 0;
+}
+
 /**
  * Helper checks if we can process main and overlay pixel formats
  */
@@ -151,10 +228,8 @@  static int overlay_cuda_blend(FFFrameSync *fs)
     CUcontext dummy, cuda_ctx = ctx->hwctx->cuda_ctx;
 
     AVFrame *input_main, *input_overlay;
-    const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(inlink->format);
 
-    int hsub = pix_desc->log2_chroma_w;
-    int vsub = pix_desc->log2_chroma_h;
+    int pos = 0;
 
     ctx->cu_ctx = cuda_ctx;
 
@@ -183,8 +258,17 @@  static int overlay_cuda_blend(FFFrameSync *fs)
         return ret;
     }
 
-    ctx->x_position &= (1 << hsub) - 1;
-    ctx->y_position &= (1 << vsub) - 1;
+    pos = input_main->pkt_pos;
+    ctx->var_values[VAR_N] = inlink->frame_count_out;
+    ctx->var_values[VAR_T] = input_main->pts == AV_NOPTS_VALUE ?
+        NAN : input_main->pts * av_q2d(inlink->time_base);
+    ctx->var_values[VAR_POS] = pos == -1 ? NAN : pos;
+    ctx->var_values[VAR_OVERLAY_W] = ctx->var_values[VAR_OW] = input_overlay->width;
+    ctx->var_values[VAR_OVERLAY_H] = ctx->var_values[VAR_OH] = input_overlay->height;
+    ctx->var_values[VAR_MAIN_W   ] = ctx->var_values[VAR_MW] = input_main->width;
+    ctx->var_values[VAR_MAIN_H   ] = ctx->var_values[VAR_MH] = input_main->height;
+
+    eval_expr(avctx);
 
     // overlay first plane
 
@@ -238,6 +322,32 @@  static int overlay_cuda_blend(FFFrameSync *fs)
     return ff_filter_frame(outlink, input_main);
 }
 
+static int config_input_overlay(AVFilterLink *inlink)
+{
+    AVFilterContext *ctx  = inlink->dst;
+    OverlayCUDAContext  *s = inlink->dst->priv;
+    int ret;
+
+
+    /* Finish the configuration by evaluating the expressions
+       now when both inputs are configured. */
+    s->var_values[VAR_MAIN_W   ] = s->var_values[VAR_MW] = ctx->inputs[MAIN   ]->w;
+    s->var_values[VAR_MAIN_H   ] = s->var_values[VAR_MH] = ctx->inputs[MAIN   ]->h;
+    s->var_values[VAR_OVERLAY_W] = s->var_values[VAR_OW] = ctx->inputs[OVERLAY]->w;
+    s->var_values[VAR_OVERLAY_H] = s->var_values[VAR_OH] = ctx->inputs[OVERLAY]->h;
+    s->var_values[VAR_X]     = NAN;
+    s->var_values[VAR_Y]     = NAN;
+    s->var_values[VAR_N]     = 0;
+    s->var_values[VAR_T]     = NAN;
+    s->var_values[VAR_POS]   = NAN;
+
+    if ((ret = set_expr(&s->x_pexpr,      s->x_expr,      "x",      ctx)) < 0 ||
+        (ret = set_expr(&s->y_pexpr,      s->y_expr,      "y",      ctx)) < 0)
+        return ret;
+
+    return 0;
+}
+
 /**
  * Initialize overlay_cuda
  */
@@ -266,6 +376,8 @@  static av_cold void overlay_cuda_uninit(AVFilterContext *avctx)
         CHECK_CU(cu->cuCtxPopCurrent(&dummy));
     }
 
+    av_expr_free(ctx->x_pexpr); ctx->x_pexpr = NULL;
+    av_expr_free(ctx->y_pexpr); ctx->y_pexpr = NULL;
     av_buffer_unref(&ctx->hw_device_ctx);
     ctx->hwctx = NULL;
 }
@@ -405,10 +517,8 @@  static int overlay_cuda_config_output(AVFilterLink *outlink)
 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
 
 static const AVOption overlay_cuda_options[] = {
-    { "x", "Overlay x position",
-      OFFSET(x_position), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, .flags = FLAGS },
-    { "y", "Overlay y position",
-      OFFSET(y_position), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, .flags = FLAGS },
+    { "x", "set the x expression of overlay", OFFSET(x_expr), AV_OPT_TYPE_STRING, {.str = "0"}, 0, 0, FLAGS },
+    { "y", "set the y expression of overlay", OFFSET(y_expr), AV_OPT_TYPE_STRING, {.str = "0"}, 0, 0, FLAGS },
     { "eof_action", "Action to take when encountering EOF from secondary input ",
         OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT },
         EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, "eof_action" },
@@ -430,6 +540,7 @@  static const AVFilterPad overlay_cuda_inputs[] = {
     {
         .name         = "overlay",
         .type         = AVMEDIA_TYPE_VIDEO,
+        .config_props = config_input_overlay,
     },
     { NULL }
 };