| Message ID | 20220808000124.12794-1-jamrial@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel] avfilter/framesync: add a new option to set how to sync streams based on secondary input timestamps | expand |
| Context | Check | Description |
|---|---|---|
| yinshiyou/make_loongarch64 | success | Make finished |
| yinshiyou/make_fate_loongarch64 | success | Make fate finished |
| andriy/make_x86 | success | Make finished |
| andriy/make_fate_x86 | success | Make fate finished |
James Almer (12022-08-07): > Include two values for it, a default one that sets/keeps the current behavior, > where the frame event generated by the primary input will have a timestamp > equal or higher than frames in secondary input, plus a new one where the > secondary input frame will be that with the absolute closest timestamp to that > of the frame event one. > > Addresses ticket #9689, where the new optional behavior produces better frame > syncronization. I do not find a better way of doing it. Thanks for the patch. > > Signed-off-by: James Almer <jamrial@gmail.com> > --- > libavfilter/framesync.c | 19 +++++++++++++++++++ > libavfilter/framesync.h | 23 +++++++++++++++++++++++ > 2 files changed, 42 insertions(+) > > diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c > index 7510550d8e..bd784f6e0b 100644 > --- a/libavfilter/framesync.c > +++ b/libavfilter/framesync.c > @@ -42,6 +42,13 @@ static const AVOption framesync_options[] = { > { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" }, > { "shortest", "force termination when the shortest input terminates", OFFSET(opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, > { "repeatlast", "extend last frame of secondary streams beyond EOF", OFFSET(opt_repeatlast), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS }, > + { "ts_sync_mode", "How strictly to sync streams based on secondary input timestamps", > + OFFSET(opt_ts_sync_mode), AV_OPT_TYPE_INT, { .i64 = TS_DEFAULT }, > + TS_DEFAULT, TS_NEAREST, .flags = FLAGS, "ts_sync_mode" }, > + { "default", "Frame from secondary input with the nearest lower or equal timestamp to the primary input frame", > + 0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, "ts_sync_mode" }, > + { "nearest", "Frame from secondary input with the absolute nearest timestamp to the primary input frame", > + 0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, "ts_sync_mode" }, This should be a per-stream options, but our options parsing is not powerful enough for that. And since options are strings and my projects to have a good strings API are being blocked by somebody whose arguments can be summarized as “I never deal with strings, I do not know about our use of strings, therefore we do not need this API”, we will not have a better options parsing system in the foreseeable future. But don't mind me, I'm just being bitter that this person does not extend to me the respect in abilities and trust in vision he enjoys. > { NULL } > }; > static const AVClass framesync_class = { > @@ -110,6 +117,14 @@ static void framesync_sync_level_update(FFFrameSync *fs) > av_assert0(level <= fs->sync_level); > if (level < fs->sync_level) > av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level); > + if (fs->opt_ts_sync_mode > TS_DEFAULT) { > + for (i = 0; i < fs->nb_in; i++) { > + if (fs->in[i].sync < level) > + fs->in[i].ts_mode = fs->opt_ts_sync_mode; > + else > + fs->in[i].ts_mode = TS_DEFAULT; > + } > + } > if (level) > fs->sync_level = level; > else > @@ -187,6 +202,10 @@ static int framesync_advance(FFFrameSync *fs) > } > for (i = 0; i < fs->nb_in; i++) { > if (fs->in[i].pts_next == pts || > + (fs->in[i].ts_mode == TS_NEAREST && > + fs->in[i].have_next && > + fs->in[i].pts_next != INT64_MAX && fs->in[i].pts != AV_NOPTS_VALUE && > + FFABS(pts - fs->in[i].pts_next) < FFABS(pts - fs->in[i].pts)) || fs->in[i].pts_next - pts < pts - fs->in[i].pts No need for a FFABS since we know how they compare. > (fs->in[i].before == EXT_INFINITY && > fs->in[i].state == STATE_BOF)) { > av_frame_free(&fs->in[i].frame); > diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h > index a246d2d1e5..233f50a0eb 100644 > --- a/libavfilter/framesync.h > +++ b/libavfilter/framesync.h > @@ -75,6 +75,27 @@ enum FFFrameSyncExtMode { > EXT_INFINITY, > }; > > +/** > + * Timestamp syncronization mode > + * > + * Describe how the frames of a stream are syncronized based on timestamp > + * distance. > + */ > +enum FFFrameTSSyncMode { > + > + /** > + * Sync to frames from secondary input with the nearest, lower or equal > + * timestamp to the frame event one. > + */ > + TS_DEFAULT, > + > + /** > + * Sync to frames from secondary input with the absolute nearest timestamp > + * to the frame event one. > + */ > + TS_NEAREST, > +}; > + > /** > * Input stream structure > */ > @@ -138,6 +159,7 @@ typedef struct FFFrameSyncIn { > */ > unsigned sync; > > + enum FFFrameTSSyncMode ts_mode; > } FFFrameSyncIn; > > /** > @@ -205,6 +227,7 @@ typedef struct FFFrameSync { > int opt_repeatlast; > int opt_shortest; > int opt_eof_action; > + int opt_ts_sync_mode; > > } FFFrameSync; > Ok except for the remark above. Regards,
diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c index 7510550d8e..bd784f6e0b 100644 --- a/libavfilter/framesync.c +++ b/libavfilter/framesync.c @@ -42,6 +42,13 @@ static const AVOption framesync_options[] = { { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" }, { "shortest", "force termination when the shortest input terminates", OFFSET(opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, { "repeatlast", "extend last frame of secondary streams beyond EOF", OFFSET(opt_repeatlast), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS }, + { "ts_sync_mode", "How strictly to sync streams based on secondary input timestamps", + OFFSET(opt_ts_sync_mode), AV_OPT_TYPE_INT, { .i64 = TS_DEFAULT }, + TS_DEFAULT, TS_NEAREST, .flags = FLAGS, "ts_sync_mode" }, + { "default", "Frame from secondary input with the nearest lower or equal timestamp to the primary input frame", + 0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, "ts_sync_mode" }, + { "nearest", "Frame from secondary input with the absolute nearest timestamp to the primary input frame", + 0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, "ts_sync_mode" }, { NULL } }; static const AVClass framesync_class = { @@ -110,6 +117,14 @@ static void framesync_sync_level_update(FFFrameSync *fs) av_assert0(level <= fs->sync_level); if (level < fs->sync_level) av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level); + if (fs->opt_ts_sync_mode > TS_DEFAULT) { + for (i = 0; i < fs->nb_in; i++) { + if (fs->in[i].sync < level) + fs->in[i].ts_mode = fs->opt_ts_sync_mode; + else + fs->in[i].ts_mode = TS_DEFAULT; + } + } if (level) fs->sync_level = level; else @@ -187,6 +202,10 @@ static int framesync_advance(FFFrameSync *fs) } for (i = 0; i < fs->nb_in; i++) { if (fs->in[i].pts_next == pts || + (fs->in[i].ts_mode == TS_NEAREST && + fs->in[i].have_next && + fs->in[i].pts_next != INT64_MAX && fs->in[i].pts != AV_NOPTS_VALUE && + FFABS(pts - fs->in[i].pts_next) < FFABS(pts - fs->in[i].pts)) || (fs->in[i].before == EXT_INFINITY && fs->in[i].state == STATE_BOF)) { av_frame_free(&fs->in[i].frame); diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h index a246d2d1e5..233f50a0eb 100644 --- a/libavfilter/framesync.h +++ b/libavfilter/framesync.h @@ -75,6 +75,27 @@ enum FFFrameSyncExtMode { EXT_INFINITY, }; +/** + * Timestamp syncronization mode + * + * Describe how the frames of a stream are syncronized based on timestamp + * distance. + */ +enum FFFrameTSSyncMode { + + /** + * Sync to frames from secondary input with the nearest, lower or equal + * timestamp to the frame event one. + */ + TS_DEFAULT, + + /** + * Sync to frames from secondary input with the absolute nearest timestamp + * to the frame event one. + */ + TS_NEAREST, +}; + /** * Input stream structure */ @@ -138,6 +159,7 @@ typedef struct FFFrameSyncIn { */ unsigned sync; + enum FFFrameTSSyncMode ts_mode; } FFFrameSyncIn; /** @@ -205,6 +227,7 @@ typedef struct FFFrameSync { int opt_repeatlast; int opt_shortest; int opt_eof_action; + int opt_ts_sync_mode; } FFFrameSync;
Include two values for it, a default one that sets/keeps the current behavior, where the frame event generated by the primary input will have a timestamp equal or higher than frames in secondary input, plus a new one where the secondary input frame will be that with the absolute closest timestamp to that of the frame event one. Addresses ticket #9689, where the new optional behavior produces better frame syncronization. Signed-off-by: James Almer <jamrial@gmail.com> --- libavfilter/framesync.c | 19 +++++++++++++++++++ libavfilter/framesync.h | 23 +++++++++++++++++++++++ 2 files changed, 42 insertions(+)