@@ -16,6 +16,7 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#include "libavutil/avassert.h"
#include "libavutil/eval.h"
#include "libavutil/file.h"
#include "libavutil/opt.h"
@@ -26,6 +27,7 @@
#include <libplacebo/renderer.h>
#include <libplacebo/utils/libav.h>
+#include <libplacebo/utils/frame_queue.h>
#include <libplacebo/vulkan.h>
/* Backwards compatibility with older libplacebo */
@@ -122,7 +124,8 @@ typedef struct LibplaceboContext {
pl_vulkan vulkan;
pl_gpu gpu;
pl_renderer renderer;
- pl_tex tex[8];
+ pl_queue queue;
+ pl_tex tex[4];
/* settings */
char *out_format_string;
@@ -392,6 +395,7 @@ static int update_settings(AVFilterContext *ctx)
.num_hooks = s->num_hooks,
.skip_anti_aliasing = s->skip_aa,
+ .skip_caching_single_frame = true,
.polar_cutoff = s->polar_cutoff,
.disable_linear_scaling = s->disable_linear,
.disable_builtin_scalers = s->disable_builtin,
@@ -518,6 +522,7 @@ static int init_vulkan(AVFilterContext *avctx)
/* Create the renderer */
s->gpu = s->vulkan->gpu;
s->renderer = pl_renderer_create(s->log, s->gpu);
+ s->queue = pl_queue_create(s->gpu);
/* Parse the user shaders, if requested */
if (s->shader_bin_len)
@@ -544,6 +549,7 @@ static void libplacebo_uninit(AVFilterContext *avctx)
for (int i = 0; i < s->num_hooks; i++)
pl_mpv_user_shader_destroy(&s->hooks[i]);
pl_renderer_destroy(&s->renderer);
+ pl_queue_destroy(&s->queue);
pl_vulkan_destroy(&s->vulkan);
pl_log_destroy(&s->log);
ff_vk_uninit(&s->vkctx);
@@ -573,69 +579,92 @@ fail:
}
static void update_crops(AVFilterContext *ctx,
- struct pl_frame *image,
- struct pl_frame *target,
- const double target_pts)
+ struct pl_frame_mix *mix, struct pl_frame *target,
+ uint64_t ref_sig, double base_pts)
{
LibplaceboContext *s = ctx->priv;
- const AVFrame *in = pl_get_mapped_avframe(image);
- const AVFilterLink *inlink = ctx->inputs[0];
- const double in_pts = in->pts * av_q2d(inlink->time_base);
-
- s->var_values[VAR_IN_T] = s->var_values[VAR_T] = in_pts;
- s->var_values[VAR_OUT_T] = s->var_values[VAR_OT] = target_pts;
- s->var_values[VAR_N] = inlink->frame_count_out;
-
- /* Clear these explicitly to avoid leaking previous frames' state */
- s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] = NAN;
- s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] = NAN;
- s->var_values[VAR_POS_W] = s->var_values[VAR_PW] = NAN;
- s->var_values[VAR_POS_H] = s->var_values[VAR_PH] = NAN;
-
- /* Evaluate crop/pos dimensions first, and placement second */
- s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] =
- av_expr_eval(s->crop_w_pexpr, s->var_values, NULL);
- s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] =
- av_expr_eval(s->crop_h_pexpr, s->var_values, NULL);
- s->var_values[VAR_POS_W] = s->var_values[VAR_PW] =
- av_expr_eval(s->pos_w_pexpr, s->var_values, NULL);
- s->var_values[VAR_POS_H] = s->var_values[VAR_PH] =
- av_expr_eval(s->pos_h_pexpr, s->var_values, NULL);
-
- image->crop.x0 = av_expr_eval(s->crop_x_pexpr, s->var_values, NULL);
- image->crop.y0 = av_expr_eval(s->crop_y_pexpr, s->var_values, NULL);
- image->crop.x1 = image->crop.x0 + s->var_values[VAR_CROP_W];
- image->crop.y1 = image->crop.y0 + s->var_values[VAR_CROP_H];
-
- target->crop.x0 = av_expr_eval(s->pos_x_pexpr, s->var_values, NULL);
- target->crop.y0 = av_expr_eval(s->pos_y_pexpr, s->var_values, NULL);
- target->crop.x1 = target->crop.x0 + s->var_values[VAR_POS_W];
- target->crop.y1 = target->crop.y0 + s->var_values[VAR_POS_H];
-
- if (s->target_sar.num) {
- float aspect = pl_rect2df_aspect(&target->crop) * av_q2d(s->target_sar);
- pl_rect2df_aspect_set(&target->crop, aspect, s->pad_crop_ratio);
+
+ for (int i = 0; i < mix->num_frames; i++) {
+ // Mutate the `pl_frame.crop` fields in-place. This is fine because we
+ // own the entire pl_queue, and hence, the pointed-at frames.
+ struct pl_frame *image = (struct pl_frame *) mix->frames[i];
+ double image_pts = base_pts + mix->timestamps[i];
+
+ /* Update dynamic variables */
+ s->var_values[VAR_IN_T] = s->var_values[VAR_T] = image_pts;
+ s->var_values[VAR_OUT_T] = s->var_values[VAR_OT] = base_pts;
+ s->var_values[VAR_N] = ctx->outputs[0]->frame_count_out;
+
+ /* Clear these explicitly to avoid leaking previous frames' state */
+ s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] = NAN;
+ s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] = NAN;
+ s->var_values[VAR_POS_W] = s->var_values[VAR_PW] = NAN;
+ s->var_values[VAR_POS_H] = s->var_values[VAR_PH] = NAN;
+
+ /* Compute dimensions first and placement second */
+ s->var_values[VAR_CROP_W] = s->var_values[VAR_CW] =
+ av_expr_eval(s->crop_w_pexpr, s->var_values, NULL);
+ s->var_values[VAR_CROP_H] = s->var_values[VAR_CH] =
+ av_expr_eval(s->crop_h_pexpr, s->var_values, NULL);
+ s->var_values[VAR_POS_W] = s->var_values[VAR_PW] =
+ av_expr_eval(s->pos_w_pexpr, s->var_values, NULL);
+ s->var_values[VAR_POS_H] = s->var_values[VAR_PH] =
+ av_expr_eval(s->pos_h_pexpr, s->var_values, NULL);
+
+ image->crop.x0 = av_expr_eval(s->crop_x_pexpr, s->var_values, NULL);
+ image->crop.y0 = av_expr_eval(s->crop_y_pexpr, s->var_values, NULL);
+ image->crop.x1 = image->crop.x0 + s->var_values[VAR_CROP_W];
+ image->crop.y1 = image->crop.y0 + s->var_values[VAR_CROP_H];
+
+ if (mix->signatures[i] == ref_sig) {
+ /* Only update the target crop once, for the 'reference' frame */
+ target->crop.x0 = av_expr_eval(s->pos_x_pexpr, s->var_values, NULL);
+ target->crop.y0 = av_expr_eval(s->pos_y_pexpr, s->var_values, NULL);
+ target->crop.x1 = target->crop.x0 + s->var_values[VAR_POS_W];
+ target->crop.y1 = target->crop.y0 + s->var_values[VAR_POS_H];
+
+ if (s->target_sar.num) {
+ float aspect = pl_rect2df_aspect(&target->crop) * av_q2d(s->target_sar);
+ pl_rect2df_aspect_set(&target->crop, aspect, s->pad_crop_ratio);
+ }
+ }
}
}
-/* Construct and emit an output frame for `image` */
-static int output_frame(AVFilterContext *ctx, struct pl_frame *image)
+/* Construct and emit an output frame for a given frame mix */
+static int output_frame_mix(AVFilterContext *ctx,
+ struct pl_frame_mix *mix,
+ int64_t pts)
{
int err = 0, ok, changed_csp;
LibplaceboContext *s = ctx->priv;
AVFilterLink *outlink = ctx->outputs[0];
const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(outlink->format);
- AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
- const AVFrame *in = pl_get_mapped_avframe(image);
struct pl_frame target;
+ const AVFrame *ref;
+ AVFrame *out;
+ uint64_t ref_sig;
+ if (!mix->num_frames)
+ return 0;
+
+ out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
if (!out)
return AVERROR(ENOMEM);
- RET(av_frame_copy_props(out, in));
+ /* Use the last frame before current PTS value as reference */
+ for (int i = 0; i < mix->num_frames; i++) {
+ if (i && mix->timestamps[i] > 0.0f)
+ break;
+ ref = pl_get_mapped_avframe(mix->frames[i]);
+ ref_sig = mix->signatures[i];
+ }
+
+ RET(av_frame_copy_props(out, ref));
+ out->pts = pts;
out->width = outlink->w;
out->height = outlink->h;
- if (s->apply_dovi && av_frame_get_side_data(in, AV_FRAME_DATA_DOVI_METADATA)) {
+ if (s->apply_dovi && av_frame_get_side_data(ref, AV_FRAME_DATA_DOVI_METADATA)) {
/* Output of dovi reshaping is always BT.2020+PQ, so infer the correct
* output colorspace defaults */
out->colorspace = AVCOL_SPC_BT2020_NCL;
@@ -652,10 +681,10 @@ static int output_frame(AVFilterContext *ctx, struct pl_frame *image)
if (s->color_primaries >= 0)
out->color_primaries = s->color_primaries;
- changed_csp = in->colorspace != out->colorspace ||
- in->color_range != out->color_range ||
- in->color_trc != out->color_trc ||
- in->color_primaries != out->color_primaries;
+ changed_csp = ref->colorspace != out->colorspace ||
+ ref->color_range != out->color_range ||
+ ref->color_trc != out->color_trc ||
+ ref->color_primaries != out->color_primaries;
/* Strip side data if no longer relevant */
if (changed_csp) {
@@ -677,15 +706,15 @@ static int output_frame(AVFilterContext *ctx, struct pl_frame *image)
.map_dovi = false,
));
} else {
- ok = pl_frame_recreate_from_avframe(s->gpu, &target, s->tex + 4, out);
+ ok = pl_frame_recreate_from_avframe(s->gpu, &target, s->tex, out);
}
if (!ok) {
err = AVERROR_EXTERNAL;
goto fail;
}
- update_crops(ctx, image, &target, out->pts * av_q2d(outlink->time_base));
- pl_render_image(s->renderer, image, &target, &s->params);
+ update_crops(ctx, mix, &target, ref_sig, out->pts * av_q2d(outlink->time_base));
+ pl_render_image_mix(s->renderer, mix, &target, &s->params);
if (outdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
pl_unmap_avframe(s->gpu, &target);
@@ -700,33 +729,78 @@ fail:
return err;
}
+static bool map_frame(pl_gpu gpu, pl_tex *tex,
+ const struct pl_source_frame *src,
+ struct pl_frame *out)
+{
+ AVFrame *avframe = src->frame_data;
+ LibplaceboContext *s = avframe->opaque;
+ bool ok = pl_map_avframe_ex(gpu, out, pl_avframe_params(
+ .frame = avframe,
+ .tex = tex,
+ .map_dovi = s->apply_dovi,
+ ));
+
+ if (!s->apply_filmgrain)
+ out->film_grain.type = PL_FILM_GRAIN_NONE;
+
+ av_frame_free(&avframe);
+ return ok;
+}
+
+static void unmap_frame(pl_gpu gpu, struct pl_frame *frame,
+ const struct pl_source_frame *src)
+{
+ pl_unmap_avframe(gpu, frame);
+}
+
+static void discard_frame(const struct pl_source_frame *src)
+{
+ AVFrame *avframe = src->frame_data;
+ av_frame_free(&avframe);
+}
+
static int filter_frame(AVFilterLink *link, AVFrame *in)
{
- int ret, ok;
AVFilterContext *ctx = link->dst;
LibplaceboContext *s = ctx->priv;
- struct pl_frame image;
+ AVFilterLink *outlink = ctx->outputs[0];
+ enum pl_queue_status status;
+ struct pl_frame_mix mix;
pl_log_level_update(s->log, get_log_level());
- /* Map input frame */
- ok = pl_map_avframe_ex(s->gpu, &image, pl_avframe_params(
- .frame = in,
- .tex = s->tex,
- .map_dovi = s->apply_dovi,
+ /* Push input frame */
+ in->opaque = s;
+ pl_queue_push(s->queue, &(struct pl_source_frame) {
+ .pts = in->pts * av_q2d(link->time_base),
+ .duration = in->duration * av_q2d(link->time_base),
+ .first_field = pl_field_from_avframe(in),
+ .frame_data = in,
+ .map = map_frame,
+ .unmap = unmap_frame,
+ .discard = discard_frame,
+ });
+
+ /* Immediately return an output frame for the same PTS */
+ av_assert1(!av_cmp_q(link->time_base, outlink->time_base));
+ status = pl_queue_update(s->queue, &mix, pl_queue_params(
+ .pts = in->pts * av_q2d(outlink->time_base),
+ .radius = pl_frame_mix_radius(&s->params),
+ .vsync_duration = av_q2d(av_inv_q(outlink->frame_rate)),
));
- av_frame_free(&in);
-
- if (!s->apply_filmgrain)
- image.film_grain.type = PL_FILM_GRAIN_NONE;
- if (!ok)
+ switch (status) {
+ case PL_QUEUE_MORE: // TODO: switch to activate() and handle properly
+ case PL_QUEUE_OK:
+ return output_frame_mix(ctx, &mix, in->pts);
+ case PL_QUEUE_EOF:
+ return 0;
+ case PL_QUEUE_ERR:
return AVERROR_EXTERNAL;
+ }
- ret = output_frame(ctx, &image);
-
- pl_unmap_avframe(s->gpu, &image);
- return ret;
+ return AVERROR_BUG;
}
static int libplacebo_query_format(AVFilterContext *ctx)
From: Niklas Haas <git@haasn.dev> This does not leverage any immediate benefits, but refactors and prepares the codebase for upcoming changes, which will include the ability to do deinterlacing and resampling (frame mixing). --- libavfilter/vf_libplacebo.c | 214 ++++++++++++++++++++++++------------ 1 file changed, 144 insertions(+), 70 deletions(-)