| Message ID | 20240606115100.44233-1-hiccupzhu@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel] avfilter: add sdlvsink for video display | expand |
| Context | Check | Description |
|---|---|---|
| andriy/make_x86 | success | Make finished |
| andriy/make_fate_x86 | success | Make fate finished |
> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote: I’m afraid this has the same issue as libavdevice/sdl[1]. [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/ > > Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com> > --- > configure | 1 + > libavfilter/Makefile | 1 + > libavfilter/allfilters.c | 1 + > libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++ > 4 files changed, 145 insertions(+) > create mode 100644 libavfilter/vsink_sdlvsink.c > > diff --git a/configure b/configure > index 6c5b8aab9a..968b5c8912 100755 > --- a/configure > +++ b/configure > @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx" > xstack_qsv_filter_select="qsvvpp" > pad_vaapi_filter_deps="vaapi_1" > drawbox_vaapi_filter_deps="vaapi_1" > +sdlvsink_filter_deps="sdl2" > > # examples > avio_http_serve_files_deps="avformat avutil fork" > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 5992fd161f..feac6b464d 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER) += vsrc_testsrc.o > OBJS-$(CONFIG_ZONEPLATE_FILTER) += vsrc_testsrc.o > > OBJS-$(CONFIG_NULLSINK_FILTER) += vsink_nullsink.o > +OBJS-$(CONFIG_SDLVSINK_FILTER) += vsink_sdlvsink.o > > # multimedia filters > OBJS-$(CONFIG_A3DSCOPE_FILTER) += avf_a3dscope.o > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > index c532682fc2..d5f942ffb5 100644 > --- a/libavfilter/allfilters.c > +++ b/libavfilter/allfilters.c > @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc; > extern const AVFilter ff_vsrc_zoneplate; > > extern const AVFilter ff_vsink_nullsink; > +extern const AVFilter ff_vsink_sdlvsink; > > /* multimedia filters */ > extern const AVFilter ff_avf_a3dscope; > diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c > new file mode 100644 > index 0000000000..630f719c7e > --- /dev/null > +++ b/libavfilter/vsink_sdlvsink.c > @@ -0,0 +1,142 @@ > +/* > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#include "avfilter.h" > +#include "internal.h" > +#include "libavutil/internal.h" > +#include <SDL2/SDL.h> > +#include <unistd.h> > + > +typedef struct { > + SDL_Window *window; > + SDL_Renderer *render; > + SDL_Texture *texture; > +} SDLVideoContext; > + > +static int init(AVFilterContext *ctx) > +{ > + if (SDL_Init(SDL_INIT_VIDEO) < 0) > + { > + av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError()); > + return -ENOMEM; > + } > + > + SDL_PollEvent(NULL); > + SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); > + > + return 0; > +} > + > +static void uninit(AVFilterContext *ctx) > +{ > + SDLVideoContext *sink = ctx->priv; > + > + if (sink->texture) { > + SDL_DestroyTexture(sink->texture); > + sink->texture = NULL; > + } > + > + if (sink->render) { > + SDL_DestroyRenderer(sink->render); > + sink->render = NULL; > + } > + > + if (sink->window) { > + SDL_DestroyWindow(sink->window); > + sink->window = NULL; > + } > + > + SDL_Quit(); > +} > + > +static int filter_frame(AVFilterLink *link, AVFrame *frame) > +{ > + AVFilterContext *ctx = link->dst; > + SDLVideoContext *sink = ctx->priv; > + SDL_RendererInfo renderer_info; > + > + if (!sink->window) { > + sink->window = SDL_CreateWindow("YUV Player", > + SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, > + frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE); > + if (!sink->window) { > + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError()); > + return -ENOMEM; > + } > + } > + > + if (!sink->render) { > + sink->render = SDL_CreateRenderer(sink->window, -1, 0); > + if (!sink->render) { > + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError()); > + return -ENOMEM; > + } > + SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); > + > + SDL_GetRendererInfo(sink->render, &renderer_info); > + av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name); > + } > + > + if (!sink->texture) { > + sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV, > + SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height); > + if (!sink->texture) { > + av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError()); > + return -EINVAL; > + } > + } > + > + SDL_UpdateYUVTexture(sink->texture, NULL, > + frame->data[0], frame->linesize[0], > + frame->data[1], frame->linesize[1], > + frame->data[2], frame->linesize[2]); > + > + SDL_RenderClear(sink->render); > + > + SDL_RenderCopy(sink->render, sink->texture, NULL, NULL); > + SDL_RenderPresent(sink->render); > + > + av_frame_free(&frame); > + > + return 0; > +} > + > +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = { > + { > + .name = "default", > + .type = AVMEDIA_TYPE_VIDEO, > + .filter_frame = filter_frame, > + }, > +}; > + > +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = { > + { > + .name = NULL, > + .type = AVMEDIA_TYPE_VIDEO, > + }, > +}; > + > +const AVFilter ff_vsink_sdlvsink = { > + .name = "sdlvsink", > + .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."), > + .priv_size = sizeof(SDLVideoContext), > + .init = init, > + .uninit = uninit, > + FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs), > + FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs), > +}; > -- > 2.34.1 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote: > > > On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote: > > > I’m afraid this has the same issue as libavdevice/sdl[1]. > > [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/ > Thank you for the reminder. This patch does not have that issue. All SDL initialization is within the filter_frame() function, so there's no problem with different threads being created. Furthermore, my expertise lies primarily in multimedia framework development. I've noticed that the libavfilter now offers a comprehensive basic framework, providing various types of filters such as src, trans, and sink within the filter-graph. There are already many src and trans type filters in the existing codebase, with rich functionalities. However, there are only two sink filters: buffersink and nullsink, which is not quite sufficient for the development of filter-graphs. Thus, I submitted this patch with the intention of enriching the types of sink filters available. Many thanks for your valuable comments. > > Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com> > --- > configure | 1 + > libavfilter/Makefile | 1 + > libavfilter/allfilters.c | 1 + > libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++ > 4 files changed, 145 insertions(+) > create mode 100644 libavfilter/vsink_sdlvsink.c > > diff --git a/configure b/configure > index 6c5b8aab9a..968b5c8912 100755 > --- a/configure > +++ b/configure > @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx" > xstack_qsv_filter_select="qsvvpp" > pad_vaapi_filter_deps="vaapi_1" > drawbox_vaapi_filter_deps="vaapi_1" > +sdlvsink_filter_deps="sdl2" > > # examples > avio_http_serve_files_deps="avformat avutil fork" > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 5992fd161f..feac6b464d 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER) += vsrc_testsrc.o > OBJS-$(CONFIG_ZONEPLATE_FILTER) += vsrc_testsrc.o > > OBJS-$(CONFIG_NULLSINK_FILTER) += vsink_nullsink.o > +OBJS-$(CONFIG_SDLVSINK_FILTER) += vsink_sdlvsink.o > > # multimedia filters > OBJS-$(CONFIG_A3DSCOPE_FILTER) += avf_a3dscope.o > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > index c532682fc2..d5f942ffb5 100644 > --- a/libavfilter/allfilters.c > +++ b/libavfilter/allfilters.c > @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc; > extern const AVFilter ff_vsrc_zoneplate; > > extern const AVFilter ff_vsink_nullsink; > +extern const AVFilter ff_vsink_sdlvsink; > > /* multimedia filters */ > extern const AVFilter ff_avf_a3dscope; > diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c > new file mode 100644 > index 0000000000..630f719c7e > --- /dev/null > +++ b/libavfilter/vsink_sdlvsink.c > @@ -0,0 +1,142 @@ > +/* > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#include "avfilter.h" > +#include "internal.h" > +#include "libavutil/internal.h" > +#include <SDL2/SDL.h> > +#include <unistd.h> > + > +typedef struct { > + SDL_Window *window; > + SDL_Renderer *render; > + SDL_Texture *texture; > +} SDLVideoContext; > + > +static int init(AVFilterContext *ctx) > +{ > + if (SDL_Init(SDL_INIT_VIDEO) < 0) > + { > + av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError()); > + return -ENOMEM; > + } > + > + SDL_PollEvent(NULL); > + SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); > + > + return 0; > +} > + > +static void uninit(AVFilterContext *ctx) > +{ > + SDLVideoContext *sink = ctx->priv; > + > + if (sink->texture) { > + SDL_DestroyTexture(sink->texture); > + sink->texture = NULL; > + } > + > + if (sink->render) { > + SDL_DestroyRenderer(sink->render); > + sink->render = NULL; > + } > + > + if (sink->window) { > + SDL_DestroyWindow(sink->window); > + sink->window = NULL; > + } > + > + SDL_Quit(); > +} > + > +static int filter_frame(AVFilterLink *link, AVFrame *frame) > +{ > + AVFilterContext *ctx = link->dst; > + SDLVideoContext *sink = ctx->priv; > + SDL_RendererInfo renderer_info; > + > + if (!sink->window) { > + sink->window = SDL_CreateWindow("YUV Player", > + SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, > + frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE); > + if (!sink->window) { > + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError()); > + return -ENOMEM; > + } > + } > + > + if (!sink->render) { > + sink->render = SDL_CreateRenderer(sink->window, -1, 0); > + if (!sink->render) { > + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError()); > + return -ENOMEM; > + } > + SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); > + > + SDL_GetRendererInfo(sink->render, &renderer_info); > + av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name); > + } > + > + if (!sink->texture) { > + sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV, > + SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height); > + if (!sink->texture) { > + av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError()); > + return -EINVAL; > + } > + } > + > + SDL_UpdateYUVTexture(sink->texture, NULL, > + frame->data[0], frame->linesize[0], > + frame->data[1], frame->linesize[1], > + frame->data[2], frame->linesize[2]); > + > + SDL_RenderClear(sink->render); > + > + SDL_RenderCopy(sink->render, sink->texture, NULL, NULL); > + SDL_RenderPresent(sink->render); > + > + av_frame_free(&frame); > + > + return 0; > +} > + > +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = { > + { > + .name = "default", > + .type = AVMEDIA_TYPE_VIDEO, > + .filter_frame = filter_frame, > + }, > +}; > + > +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = { > + { > + .name = NULL, > + .type = AVMEDIA_TYPE_VIDEO, > + }, > +}; > + > +const AVFilter ff_vsink_sdlvsink = { > + .name = "sdlvsink", > + .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."), > + .priv_size = sizeof(SDLVideoContext), > + .init = init, > + .uninit = uninit, > + FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs), > + FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs), > +}; > -- > 2.34.1 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > >
> On Jun 7, 2024, at 10:12, Hiccup Zhu <hiccupzhu@gmail.com> wrote: > > On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote: >> >> >> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote: >> >> >> I’m afraid this has the same issue as libavdevice/sdl[1]. >> >> [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/ >> > > Thank you for the reminder. This patch does not have that issue. All > SDL initialization is within the filter_frame() function, so there's > no problem with different threads being created. Initialization and render in the same thread is one of the issues. Another issue is SDL render should be run in main thread. Maybe it’s not the duty of this filter to control which thread to be called, but this wouldn’t work with ffmpeg cmd. > > Furthermore, my expertise lies primarily in multimedia framework > development. I've noticed that the libavfilter now offers a > comprehensive basic framework, providing various types of filters such > as src, trans, and sink within the filter-graph. libavdevice/sdl should work for this case, although it’s deprecated, and it support more pixel formats than current patch. TBH, a videosink in libavfilter make sense than libavdevice. But with the reasons mentioned earlier, I don’t know how to make progress. > > There are already many src and trans type filters in the existing > codebase, with rich functionalities. However, there are only two sink > filters: buffersink and nullsink, which is not quite sufficient for > the development of filter-graphs. > > Thus, I submitted this patch with the intention of enriching the types > of sink filters available. > > Many thanks for your valuable comments. > >> >> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com> >> --- >> configure | 1 + >> libavfilter/Makefile | 1 + >> libavfilter/allfilters.c | 1 + >> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++ >> 4 files changed, 145 insertions(+) >> create mode 100644 libavfilter/vsink_sdlvsink.c >> >> diff --git a/configure b/configure >> index 6c5b8aab9a..968b5c8912 100755 >> --- a/configure >> +++ b/configure >> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx" >> xstack_qsv_filter_select="qsvvpp" >> pad_vaapi_filter_deps="vaapi_1" >> drawbox_vaapi_filter_deps="vaapi_1" >> +sdlvsink_filter_deps="sdl2" >> >> # examples >> avio_http_serve_files_deps="avformat avutil fork" >> diff --git a/libavfilter/Makefile b/libavfilter/Makefile >> index 5992fd161f..feac6b464d 100644 >> --- a/libavfilter/Makefile >> +++ b/libavfilter/Makefile >> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER) += vsrc_testsrc.o >> OBJS-$(CONFIG_ZONEPLATE_FILTER) += vsrc_testsrc.o >> >> OBJS-$(CONFIG_NULLSINK_FILTER) += vsink_nullsink.o >> +OBJS-$(CONFIG_SDLVSINK_FILTER) += vsink_sdlvsink.o >> >> # multimedia filters >> OBJS-$(CONFIG_A3DSCOPE_FILTER) += avf_a3dscope.o >> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c >> index c532682fc2..d5f942ffb5 100644 >> --- a/libavfilter/allfilters.c >> +++ b/libavfilter/allfilters.c >> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc; >> extern const AVFilter ff_vsrc_zoneplate; >> >> extern const AVFilter ff_vsink_nullsink; >> +extern const AVFilter ff_vsink_sdlvsink; >> >> /* multimedia filters */ >> extern const AVFilter ff_avf_a3dscope; >> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c >> new file mode 100644 >> index 0000000000..630f719c7e >> --- /dev/null >> +++ b/libavfilter/vsink_sdlvsink.c >> @@ -0,0 +1,142 @@ >> +/* >> + * This file is part of FFmpeg. >> + * >> + * FFmpeg is free software; you can redistribute it and/or >> + * modify it under the terms of the GNU Lesser General Public >> + * License as published by the Free Software Foundation; either >> + * version 2.1 of the License, or (at your option) any later version. >> + * >> + * FFmpeg is distributed in the hope that it will be useful, >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU >> + * Lesser General Public License for more details. >> + * >> + * You should have received a copy of the GNU Lesser General Public >> + * License along with FFmpeg; if not, write to the Free Software >> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA >> + */ >> + >> +#include "avfilter.h" >> +#include "internal.h" >> +#include "libavutil/internal.h" >> +#include <SDL2/SDL.h> >> +#include <unistd.h> >> + >> +typedef struct { >> + SDL_Window *window; >> + SDL_Renderer *render; >> + SDL_Texture *texture; >> +} SDLVideoContext; >> + >> +static int init(AVFilterContext *ctx) >> +{ >> + if (SDL_Init(SDL_INIT_VIDEO) < 0) >> + { >> + av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError()); >> + return -ENOMEM; >> + } >> + >> + SDL_PollEvent(NULL); >> + SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); >> + >> + return 0; >> +} >> + >> +static void uninit(AVFilterContext *ctx) >> +{ >> + SDLVideoContext *sink = ctx->priv; >> + >> + if (sink->texture) { >> + SDL_DestroyTexture(sink->texture); >> + sink->texture = NULL; >> + } >> + >> + if (sink->render) { >> + SDL_DestroyRenderer(sink->render); >> + sink->render = NULL; >> + } >> + >> + if (sink->window) { >> + SDL_DestroyWindow(sink->window); >> + sink->window = NULL; >> + } >> + >> + SDL_Quit(); >> +} >> + >> +static int filter_frame(AVFilterLink *link, AVFrame *frame) >> +{ >> + AVFilterContext *ctx = link->dst; >> + SDLVideoContext *sink = ctx->priv; >> + SDL_RendererInfo renderer_info; >> + >> + if (!sink->window) { >> + sink->window = SDL_CreateWindow("YUV Player", >> + SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, >> + frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE); >> + if (!sink->window) { >> + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError()); >> + return -ENOMEM; >> + } >> + } >> + >> + if (!sink->render) { >> + sink->render = SDL_CreateRenderer(sink->window, -1, 0); >> + if (!sink->render) { >> + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError()); >> + return -ENOMEM; >> + } >> + SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); >> + >> + SDL_GetRendererInfo(sink->render, &renderer_info); >> + av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name); >> + } >> + >> + if (!sink->texture) { >> + sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV, >> + SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height); >> + if (!sink->texture) { >> + av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError()); >> + return -EINVAL; >> + } >> + } >> + >> + SDL_UpdateYUVTexture(sink->texture, NULL, >> + frame->data[0], frame->linesize[0], >> + frame->data[1], frame->linesize[1], >> + frame->data[2], frame->linesize[2]); >> + >> + SDL_RenderClear(sink->render); >> + >> + SDL_RenderCopy(sink->render, sink->texture, NULL, NULL); >> + SDL_RenderPresent(sink->render); >> + >> + av_frame_free(&frame); >> + >> + return 0; >> +} >> + >> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = { >> + { >> + .name = "default", >> + .type = AVMEDIA_TYPE_VIDEO, >> + .filter_frame = filter_frame, >> + }, >> +}; >> + >> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = { >> + { >> + .name = NULL, >> + .type = AVMEDIA_TYPE_VIDEO, >> + }, >> +}; >> + >> +const AVFilter ff_vsink_sdlvsink = { >> + .name = "sdlvsink", >> + .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."), >> + .priv_size = sizeof(SDLVideoContext), >> + .init = init, >> + .uninit = uninit, >> + FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs), >> + FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs), >> +}; >> -- >> 2.34.1 >> >> _______________________________________________ >> ffmpeg-devel mailing list >> ffmpeg-devel@ffmpeg.org >> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> >> To unsubscribe, visit link above, or email >> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >> >>
On Fri, 7 Jun 2024 at 10:32, Zhao Zhili <quinkblack@foxmail.com> wrote: > > > > > On Jun 7, 2024, at 10:12, Hiccup Zhu <hiccupzhu@gmail.com> wrote: > > > > On Thu, 6 Jun 2024 at 20:20, Zhao Zhili <quinkblack@foxmail.com> wrote: > >> > >> > >> On Jun 6, 2024, at 19:51, Shiqi Zhu <hiccupzhu@gmail.com> wrote: > >> > >> > >> I’m afraid this has the same issue as libavdevice/sdl[1]. > >> > >> [1] https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230918063728.198377-1-haihao.xiang@intel.com/ > >> > > > > Thank you for the reminder. This patch does not have that issue. All > > SDL initialization is within the filter_frame() function, so there's > > no problem with different threads being created. > > Initialization and render in the same thread is one of the issues. Another > issue is SDL render should be run in main thread. Maybe it’s not the > duty of this filter to control which thread to be called, but this wouldn’t > work with ffmpeg cmd. > Yes, I agree with your opinion. The control of the filter should be left to the application, and the ffmpeg command should only serve the purpose of demonstrating functionality. > > > > Furthermore, my expertise lies primarily in multimedia framework > > development. I've noticed that the libavfilter now offers a > > comprehensive basic framework, providing various types of filters such > > as src, trans, and sink within the filter-graph. > > libavdevice/sdl should work for this case, although it’s deprecated, > and it support more pixel formats than current patch. > Of course, in the next patch, I will expand the pixel format, provided that this sink-filter is meaningful for the entire project and can be merged into the master branch. > TBH, a videosink in libavfilter make sense than libavdevice. But with > the reasons mentioned earlier, I don’t know how to make progress. > Thanks you for your comments. I believe there are many experienced engineers in the community with whom we can collectively discuss this issue and then push the patch forward together. > > > > There are already many src and trans type filters in the existing > > codebase, with rich functionalities. However, there are only two sink > > filters: buffersink and nullsink, which is not quite sufficient for > > the development of filter-graphs. > > > > Thus, I submitted this patch with the intention of enriching the types > > of sink filters available. > > > > Many thanks for your valuable comments. > > > >> > >> Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com> > >> --- > >> configure | 1 + > >> libavfilter/Makefile | 1 + > >> libavfilter/allfilters.c | 1 + > >> libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++ > >> 4 files changed, 145 insertions(+) > >> create mode 100644 libavfilter/vsink_sdlvsink.c > >> > >> diff --git a/configure b/configure > >> index 6c5b8aab9a..968b5c8912 100755 > >> --- a/configure > >> +++ b/configure > >> @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx" > >> xstack_qsv_filter_select="qsvvpp" > >> pad_vaapi_filter_deps="vaapi_1" > >> drawbox_vaapi_filter_deps="vaapi_1" > >> +sdlvsink_filter_deps="sdl2" > >> > >> # examples > >> avio_http_serve_files_deps="avformat avutil fork" > >> diff --git a/libavfilter/Makefile b/libavfilter/Makefile > >> index 5992fd161f..feac6b464d 100644 > >> --- a/libavfilter/Makefile > >> +++ b/libavfilter/Makefile > >> @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER) += vsrc_testsrc.o > >> OBJS-$(CONFIG_ZONEPLATE_FILTER) += vsrc_testsrc.o > >> > >> OBJS-$(CONFIG_NULLSINK_FILTER) += vsink_nullsink.o > >> +OBJS-$(CONFIG_SDLVSINK_FILTER) += vsink_sdlvsink.o > >> > >> # multimedia filters > >> OBJS-$(CONFIG_A3DSCOPE_FILTER) += avf_a3dscope.o > >> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > >> index c532682fc2..d5f942ffb5 100644 > >> --- a/libavfilter/allfilters.c > >> +++ b/libavfilter/allfilters.c > >> @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc; > >> extern const AVFilter ff_vsrc_zoneplate; > >> > >> extern const AVFilter ff_vsink_nullsink; > >> +extern const AVFilter ff_vsink_sdlvsink; > >> > >> /* multimedia filters */ > >> extern const AVFilter ff_avf_a3dscope; > >> diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c > >> new file mode 100644 > >> index 0000000000..630f719c7e > >> --- /dev/null > >> +++ b/libavfilter/vsink_sdlvsink.c > >> @@ -0,0 +1,142 @@ > >> +/* > >> + * This file is part of FFmpeg. > >> + * > >> + * FFmpeg is free software; you can redistribute it and/or > >> + * modify it under the terms of the GNU Lesser General Public > >> + * License as published by the Free Software Foundation; either > >> + * version 2.1 of the License, or (at your option) any later version. > >> + * > >> + * FFmpeg is distributed in the hope that it will be useful, > >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of > >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > >> + * Lesser General Public License for more details. > >> + * > >> + * You should have received a copy of the GNU Lesser General Public > >> + * License along with FFmpeg; if not, write to the Free Software > >> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > >> + */ > >> + > >> +#include "avfilter.h" > >> +#include "internal.h" > >> +#include "libavutil/internal.h" > >> +#include <SDL2/SDL.h> > >> +#include <unistd.h> > >> + > >> +typedef struct { > >> + SDL_Window *window; > >> + SDL_Renderer *render; > >> + SDL_Texture *texture; > >> +} SDLVideoContext; > >> + > >> +static int init(AVFilterContext *ctx) > >> +{ > >> + if (SDL_Init(SDL_INIT_VIDEO) < 0) > >> + { > >> + av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError()); > >> + return -ENOMEM; > >> + } > >> + > >> + SDL_PollEvent(NULL); > >> + SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); > >> + > >> + return 0; > >> +} > >> + > >> +static void uninit(AVFilterContext *ctx) > >> +{ > >> + SDLVideoContext *sink = ctx->priv; > >> + > >> + if (sink->texture) { > >> + SDL_DestroyTexture(sink->texture); > >> + sink->texture = NULL; > >> + } > >> + > >> + if (sink->render) { > >> + SDL_DestroyRenderer(sink->render); > >> + sink->render = NULL; > >> + } > >> + > >> + if (sink->window) { > >> + SDL_DestroyWindow(sink->window); > >> + sink->window = NULL; > >> + } > >> + > >> + SDL_Quit(); > >> +} > >> + > >> +static int filter_frame(AVFilterLink *link, AVFrame *frame) > >> +{ > >> + AVFilterContext *ctx = link->dst; > >> + SDLVideoContext *sink = ctx->priv; > >> + SDL_RendererInfo renderer_info; > >> + > >> + if (!sink->window) { > >> + sink->window = SDL_CreateWindow("YUV Player", > >> + SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, > >> + frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE); > >> + if (!sink->window) { > >> + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError()); > >> + return -ENOMEM; > >> + } > >> + } > >> + > >> + if (!sink->render) { > >> + sink->render = SDL_CreateRenderer(sink->window, -1, 0); > >> + if (!sink->render) { > >> + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError()); > >> + return -ENOMEM; > >> + } > >> + SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); > >> + > >> + SDL_GetRendererInfo(sink->render, &renderer_info); > >> + av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name); > >> + } > >> + > >> + if (!sink->texture) { > >> + sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV, > >> + SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height); > >> + if (!sink->texture) { > >> + av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError()); > >> + return -EINVAL; > >> + } > >> + } > >> + > >> + SDL_UpdateYUVTexture(sink->texture, NULL, > >> + frame->data[0], frame->linesize[0], > >> + frame->data[1], frame->linesize[1], > >> + frame->data[2], frame->linesize[2]); > >> + > >> + SDL_RenderClear(sink->render); > >> + > >> + SDL_RenderCopy(sink->render, sink->texture, NULL, NULL); > >> + SDL_RenderPresent(sink->render); > >> + > >> + av_frame_free(&frame); > >> + > >> + return 0; > >> +} > >> + > >> +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = { > >> + { > >> + .name = "default", > >> + .type = AVMEDIA_TYPE_VIDEO, > >> + .filter_frame = filter_frame, > >> + }, > >> +}; > >> + > >> +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = { > >> + { > >> + .name = NULL, > >> + .type = AVMEDIA_TYPE_VIDEO, > >> + }, > >> +}; > >> + > >> +const AVFilter ff_vsink_sdlvsink = { > >> + .name = "sdlvsink", > >> + .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."), > >> + .priv_size = sizeof(SDLVideoContext), > >> + .init = init, > >> + .uninit = uninit, > >> + FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs), > >> + FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs), > >> +}; > >> -- > >> 2.34.1 > >> > >> _______________________________________________ > >> ffmpeg-devel mailing list > >> ffmpeg-devel@ffmpeg.org > >> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >> > >> To unsubscribe, visit link above, or email > >> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > >> > >> >
Hi, Le 7 juin 2024 06:33:20 GMT+03:00, Shiqi Zhu <hiccupzhu@gmail.com> a écrit : >> Initialization and render in the same thread is one of the issues. Another >> issue is SDL render should be run in main thread. Maybe it’s not the >> duty of this filter to control which thread to be called, but this wouldn’t >> work with ffmpeg cmd. >> >Yes, I agree with your opinion. The control of the filter should be >left to the application, and the ffmpeg command should only serve the >purpose of demonstrating functionality. You can't require running on the main thread. There are no ifs or buts here; libavfilter simply can't require that from the calling application or upper-layer framework and therefore it also can't warrant it to SDL. SDL is meant to be used by programs, notably games, not middleware thread-agnostic libraries like FFmpeg's libav*. I don't think that there is a point arguing *again*.
On Fri, Jun 07, 2024 at 09:45:09AM +0300, Rémi Denis-Courmont wrote: > Hi, > > Le 7 juin 2024 06:33:20 GMT+03:00, Shiqi Zhu <hiccupzhu@gmail.com> a écrit : > >> Initialization and render in the same thread is one of the issues. Another > >> issue is SDL render should be run in main thread. Maybe it’s not the > >> duty of this filter to control which thread to be called, but this wouldn’t > >> work with ffmpeg cmd. > >> > >Yes, I agree with your opinion. The control of the filter should be > >left to the application, and the ffmpeg command should only serve the > >purpose of demonstrating functionality. > > You can't require running on the main thread. There are no ifs or buts here; libavfilter simply can't require that from the calling application or upper-layer framework and therefore it also can't warrant it to SDL. We can require anything from an API that we are able to change and extend Of course we can decide not to allow such requirment even if optional but we surely _could_ add such a feature if we choose to do so thx [...]
Quoting Michael Niedermayer (2024-06-07 11:53:51) > On Fri, Jun 07, 2024 at 09:45:09AM +0300, Rémi Denis-Courmont wrote: > > Hi, > > > > Le 7 juin 2024 06:33:20 GMT+03:00, Shiqi Zhu <hiccupzhu@gmail.com> a écrit : > > >> Initialization and render in the same thread is one of the issues. Another > > >> issue is SDL render should be run in main thread. Maybe it’s not the > > >> duty of this filter to control which thread to be called, but this wouldn’t > > >> work with ffmpeg cmd. > > >> > > >Yes, I agree with your opinion. The control of the filter should be > > >left to the application, and the ffmpeg command should only serve the > > >purpose of demonstrating functionality. > > > > You can't require running on the main thread. There are no ifs or buts here; libavfilter simply can't require that from the calling application or upper-layer framework and therefore it also can't warrant it to SDL. > > We can require anything from an API that we are able to change and extend > Of course we can decide not to allow such requirment even if optional > but we surely _could_ add such a feature if we choose to do so We cannot do it in a backwards compatible way.
Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit : >We can require anything from an API that we are able to change and extend >Of course we can decide not to allow such requirment even if optional >but we surely _could_ add such a feature if we choose to do so Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice. I don't think we should add maintenance burden with code that can't be used safely.
On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> wrote: > > > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit : > >We can require anything from an API that we are able to change and extend > >Of course we can decide not to allow such requirment even if optional > >but we surely _could_ add such a feature if we choose to do so > > Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice. > > I don't think we should add maintenance burden with code that can't be used safely. Thank you all for your attention to this patch; I greatly appreciate it. I'd like to provide a brief recap of the issue we've been discussing, with the following points: 1. Addition of sink type in the filter: This enhancement is primarily based on the existing avfilter mechanism and serves as a strengthening module. Using SDL as the sink doesn't seem to be a good fit, as I'll attempt to rectify in the following patch. https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/ 2. Utilizing SDL as an implementation for the sink: Before submitting the patch, I hadn't considered many aspects. During the intense discussions, I retested the patch on different operating systems, with the following results, hoping it may assist those interested in this issue: Command: ./ffmpeg -lavfi "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink" -f null /dev/null Test1 Ubuntu 22.04 32GB test result: SUCCESS Test2 Win11 16GB test result: SUCCESS Test3 MacOS 14.4 16GB test result: FAILED Additional notes: In Test2, slight block occurred when resizing the window, possibly due to the lack of Windows message handling mechanism, solely compensated by SDL_PollEvent. I haven't delved deeper into this; if anyone knows the exact reason, please feel free to add. In Test3, direct crash occurred, indicating a segmentation fault. In conclusion: Due to differences in how graphical interfaces are implemented on different platforms, the usage of SDL varies (such as init and render must be on the main thread). Handling such differences at the application level might be more flexible and less suitable for inclusion in general-purpose libraries like ffmpeg. Finally, I'd like to extend my gratitude to everyone for their valuable contributions and insights. > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote: > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> wrote: > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit : > > >We can require anything from an API that we are able to change and extend > > >Of course we can decide not to allow such requirment even if optional > > >but we surely _could_ add such a feature if we choose to do so > > > > Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice. > > > > I don't think we should add maintenance burden with code that can't be used safely. > > Thank you all for your attention to this patch; I greatly appreciate it. > > I'd like to provide a brief recap of the issue we've been discussing, > with the following points: > > 1. Addition of sink type in the filter: > This enhancement is primarily based on the existing avfilter mechanism > and serves as a strengthening module. Using SDL as the sink doesn't > seem to be a good fit, as I'll attempt to rectify in the following > patch. > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/ > > 2. Utilizing SDL as an implementation for the sink: > Before submitting the patch, I hadn't considered many aspects. During > the intense discussions, I retested the patch on different operating > systems, with the following results, hoping it may assist those > interested in this issue: > > Command: ./ffmpeg -lavfi > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink" > -f null /dev/null In addition to this, I wonder if adding a vsink for each different output device is the correct way. We have a movie source which can be used to read from libavformat/libavdevice, probablhy we should have a movie sink to be used to write to libavformat/libavdevice, meaning that a single sink would enable access to all the supported libavformat/libavdevice outputs. I started having a look in that direction a looot of time ago. This was never finalized because I was not sure about ways to pass options to encoders and muxers, and about dealing with a variable number of outputs, I'm attaching this very old proof-of-concept patch for reference. This approach would be possibly much more complex, but should provide a single bridge in place of having a different sink for every output device or muxer.
On Wed, Jun 12, 2024 at 9:52 PM Stefano Sabatini <stefasab@gmail.com> wrote: > On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote: > > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> > wrote: > > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer < > michael@niedermayer.cc> a écrit : > > > >We can require anything from an API that we are able to change and > extend > > > >Of course we can decide not to allow such requirment even if optional > > > >but we surely _could_ add such a feature if we choose to do so > > > > > > Sure. You can also require infinite memory or an oracle be provided. > That's just not going to happen though. And having libraries depend on the > main thread is a well-documented malpractice. > > > > > > I don't think we should add maintenance burden with code that can't be > used safely. > > > > Thank you all for your attention to this patch; I greatly appreciate it. > > > > I'd like to provide a brief recap of the issue we've been discussing, > > with the following points: > > > > 1. Addition of sink type in the filter: > > This enhancement is primarily based on the existing avfilter mechanism > > and serves as a strengthening module. Using SDL as the sink doesn't > > seem to be a good fit, as I'll attempt to rectify in the following > > patch. > > > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/ > > > > 2. Utilizing SDL as an implementation for the sink: > > Before submitting the patch, I hadn't considered many aspects. During > > the intense discussions, I retested the patch on different operating > > systems, with the following results, hoping it may assist those > > interested in this issue: > > > > Command: ./ffmpeg -lavfi > > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink" > > -f null /dev/null > > In addition to this, I wonder if adding a vsink for each different > output device is the correct way. > > We have a movie source which can be used to read from > libavformat/libavdevice, probablhy we should have a movie sink to be > used to write to libavformat/libavdevice, meaning that a single sink > would enable access to all the supported libavformat/libavdevice > outputs. > > I started having a look in that direction a looot of time ago. This > was never finalized because I was not sure about ways to pass options > to encoders and muxers, and about dealing with a variable number of > outputs, I'm attaching this very old proof-of-concept patch for > reference. > > This approach would be possibly much more complex, but should provide > a single bridge in place of having a different sink for every output > device or muxer. > Pointless as real plan is to get rid of immature libavdevice API. > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >
On Thu, 13 Jun 2024 at 04:14, Paul B Mahol <onemda@gmail.com> wrote: > > On Wed, Jun 12, 2024 at 9:52 PM Stefano Sabatini <stefasab@gmail.com> wrote: > > > On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote: > > > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> > > wrote: > > > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer < > > michael@niedermayer.cc> a écrit : > > > > >We can require anything from an API that we are able to change and > > extend > > > > >Of course we can decide not to allow such requirment even if optional > > > > >but we surely _could_ add such a feature if we choose to do so > > > > > > > > Sure. You can also require infinite memory or an oracle be provided. > > That's just not going to happen though. And having libraries depend on the > > main thread is a well-documented malpractice. > > > > > > > > I don't think we should add maintenance burden with code that can't be > > used safely. > > > > > > Thank you all for your attention to this patch; I greatly appreciate it. > > > > > > I'd like to provide a brief recap of the issue we've been discussing, > > > with the following points: > > > > > > 1. Addition of sink type in the filter: > > > This enhancement is primarily based on the existing avfilter mechanism > > > and serves as a strengthening module. Using SDL as the sink doesn't > > > seem to be a good fit, as I'll attempt to rectify in the following > > > patch. > > > > > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/ > > > > > > 2. Utilizing SDL as an implementation for the sink: > > > Before submitting the patch, I hadn't considered many aspects. During > > > the intense discussions, I retested the patch on different operating > > > systems, with the following results, hoping it may assist those > > > interested in this issue: > > > > > > Command: ./ffmpeg -lavfi > > > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink" > > > -f null /dev/null > > > > In addition to this, I wonder if adding a vsink for each different > > output device is the correct way. > > > > We have a movie source which can be used to read from > > libavformat/libavdevice, probablhy we should have a movie sink to be > > used to write to libavformat/libavdevice, meaning that a single sink > > would enable access to all the supported libavformat/libavdevice > > outputs. > > > > I started having a look in that direction a looot of time ago. This > > was never finalized because I was not sure about ways to pass options > > to encoders and muxers, and about dealing with a variable number of > > outputs, I'm attaching this very old proof-of-concept patch for > > reference. > > > > This approach would be possibly much more complex, but should provide > > a single bridge in place of having a different sink for every output > > device or muxer. > > > > > Pointless as real plan is to get rid of immature libavdevice API. > > Using the libavdevice API can be somewhat confusing, especially regarding concepts like mux and demux. > > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On Thu, 13 Jun 2024 at 03:52, Stefano Sabatini <stefasab@gmail.com> wrote: > > On date Tuesday 2024-06-11 21:13:48 +0800, Shiqi Zhu wrote: > > On Fri, 7 Jun 2024 at 19:55, Rémi Denis-Courmont <remi@remlab.net> wrote: > > > Le 7 juin 2024 12:53:51 GMT+03:00, Michael Niedermayer <michael@niedermayer.cc> a écrit : > > > >We can require anything from an API that we are able to change and extend > > > >Of course we can decide not to allow such requirment even if optional > > > >but we surely _could_ add such a feature if we choose to do so > > > > > > Sure. You can also require infinite memory or an oracle be provided. That's just not going to happen though. And having libraries depend on the main thread is a well-documented malpractice. > > > > > > I don't think we should add maintenance burden with code that can't be used safely. > > > > Thank you all for your attention to this patch; I greatly appreciate it. > > > > I'd like to provide a brief recap of the issue we've been discussing, > > with the following points: > > > > 1. Addition of sink type in the filter: > > This enhancement is primarily based on the existing avfilter mechanism > > and serves as a strengthening module. Using SDL as the sink doesn't > > seem to be a good fit, as I'll attempt to rectify in the following > > patch. > > https://patchwork.ffmpeg.org/project/ffmpeg/patch/20240611130310.1131755-1-hiccupzhu@gmail.com/ > > > > 2. Utilizing SDL as an implementation for the sink: > > Before submitting the patch, I hadn't considered many aspects. During > > the intense discussions, I retested the patch on different operating > > systems, with the following results, hoping it may assist those > > interested in this issue: > > > > Command: ./ffmpeg -lavfi > > "testsrc2=size=300x200:rate=25:duration=500,format=yuv420p,sdlvsink" > > -f null /dev/null > > In addition to this, I wonder if adding a vsink for each different > output device is the correct way. > Indeed, the mechanism of libavfilter is more robust, providing essential mechanisms like format_query. These are crucial for implementing media graph mechanisms (equivalent to DirectShow's graph and GStreamer's pipeline concepts). Therefore, gradually replacing libavdevice with sink-filter should be a relatively gentle approach. > We have a movie source which can be used to read from > libavformat/libavdevice, probablhy we should have a movie sink to be > used to write to libavformat/libavdevice, meaning that a single sink > would enable access to all the supported libavformat/libavdevice > outputs. > > I started having a look in that direction a looot of time ago. This > was never finalized because I was not sure about ways to pass options > to encoders and muxers, and about dealing with a variable number of > outputs, I'm attaching this very old proof-of-concept patch for > reference. > > This approach would be possibly much more complex, but should provide > a single bridge in place of having a different sink for every output > device or muxer. > I carefully reviewed your patch, and I currently have similar functionality in my code. Here are my thoughts: Using a filter as a bridge to connect to the avdevice allows for setting the device type and basic information through parameters and options. However, the formats supported by devices are often variable. During format negotiation, it's necessary to extend AV_APP_TO_DEV_xxx for each device to implement queries for format, sampling rate, channels, etc. This can be quite labor-intensive. Using a filter as a sink can better address this issue. > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff --git a/configure b/configure index 6c5b8aab9a..968b5c8912 100755 --- a/configure +++ b/configure @@ -3977,6 +3977,7 @@ xstack_qsv_filter_deps="libmfx" xstack_qsv_filter_select="qsvvpp" pad_vaapi_filter_deps="vaapi_1" drawbox_vaapi_filter_deps="vaapi_1" +sdlvsink_filter_deps="sdl2" # examples avio_http_serve_files_deps="avformat avutil fork" diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 5992fd161f..feac6b464d 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -616,6 +616,7 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER) += vsrc_testsrc.o OBJS-$(CONFIG_ZONEPLATE_FILTER) += vsrc_testsrc.o OBJS-$(CONFIG_NULLSINK_FILTER) += vsink_nullsink.o +OBJS-$(CONFIG_SDLVSINK_FILTER) += vsink_sdlvsink.o # multimedia filters OBJS-$(CONFIG_A3DSCOPE_FILTER) += avf_a3dscope.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index c532682fc2..d5f942ffb5 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -579,6 +579,7 @@ extern const AVFilter ff_vsrc_yuvtestsrc; extern const AVFilter ff_vsrc_zoneplate; extern const AVFilter ff_vsink_nullsink; +extern const AVFilter ff_vsink_sdlvsink; /* multimedia filters */ extern const AVFilter ff_avf_a3dscope; diff --git a/libavfilter/vsink_sdlvsink.c b/libavfilter/vsink_sdlvsink.c new file mode 100644 index 0000000000..630f719c7e --- /dev/null +++ b/libavfilter/vsink_sdlvsink.c @@ -0,0 +1,142 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "avfilter.h" +#include "internal.h" +#include "libavutil/internal.h" +#include <SDL2/SDL.h> +#include <unistd.h> + +typedef struct { + SDL_Window *window; + SDL_Renderer *render; + SDL_Texture *texture; +} SDLVideoContext; + +static int init(AVFilterContext *ctx) +{ + if (SDL_Init(SDL_INIT_VIDEO) < 0) + { + av_log(ctx, AV_LOG_ERROR, "SDL2 could not initialize! %s\n", SDL_GetError()); + return -ENOMEM; + } + + SDL_PollEvent(NULL); + SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0"); + + return 0; +} + +static void uninit(AVFilterContext *ctx) +{ + SDLVideoContext *sink = ctx->priv; + + if (sink->texture) { + SDL_DestroyTexture(sink->texture); + sink->texture = NULL; + } + + if (sink->render) { + SDL_DestroyRenderer(sink->render); + sink->render = NULL; + } + + if (sink->window) { + SDL_DestroyWindow(sink->window); + sink->window = NULL; + } + + SDL_Quit(); +} + +static int filter_frame(AVFilterLink *link, AVFrame *frame) +{ + AVFilterContext *ctx = link->dst; + SDLVideoContext *sink = ctx->priv; + SDL_RendererInfo renderer_info; + + if (!sink->window) { + sink->window = SDL_CreateWindow("YUV Player", + SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, + frame->width, frame->height, SDL_WINDOW_SHOWN | SDL_WINDOW_RESIZABLE); + if (!sink->window) { + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create window %s\n", SDL_GetError()); + return -ENOMEM; + } + } + + if (!sink->render) { + sink->render = SDL_CreateRenderer(sink->window, -1, 0); + if (!sink->render) { + av_log(ctx, AV_LOG_ERROR, "SDL2 could not create render %s\n", SDL_GetError()); + return -ENOMEM; + } + SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); + + SDL_GetRendererInfo(sink->render, &renderer_info); + av_log(ctx, AV_LOG_INFO, "sdl2 renderer name:%s\n", renderer_info.name); + } + + if (!sink->texture) { + sink->texture = SDL_CreateTexture(sink->render, SDL_PIXELFORMAT_IYUV, + SDL_TEXTUREACCESS_STREAMING, frame->width, frame->height); + if (!sink->texture) { + av_log(ctx, AV_LOG_ERROR, "create texture failed %s\n", SDL_GetError()); + return -EINVAL; + } + } + + SDL_UpdateYUVTexture(sink->texture, NULL, + frame->data[0], frame->linesize[0], + frame->data[1], frame->linesize[1], + frame->data[2], frame->linesize[2]); + + SDL_RenderClear(sink->render); + + SDL_RenderCopy(sink->render, sink->texture, NULL, NULL); + SDL_RenderPresent(sink->render); + + av_frame_free(&frame); + + return 0; +} + +static const AVFilterPad avfilter_vsink_sdlvsink_inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .filter_frame = filter_frame, + }, +}; + +static const AVFilterPad avfilter_vsink_sdlvsink_outputs[] = { + { + .name = NULL, + .type = AVMEDIA_TYPE_VIDEO, + }, +}; + +const AVFilter ff_vsink_sdlvsink = { + .name = "sdlvsink", + .description = NULL_IF_CONFIG_SMALL("Do absolutely nothing with the input video."), + .priv_size = sizeof(SDLVideoContext), + .init = init, + .uninit = uninit, + FILTER_INPUTS(avfilter_vsink_sdlvsink_inputs), + FILTER_OUTPUTS(avfilter_vsink_sdlvsink_outputs), +};
Signed-off-by: Shiqi Zhu <hiccupzhu@gmail.com> --- configure | 1 + libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/vsink_sdlvsink.c | 142 +++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+) create mode 100644 libavfilter/vsink_sdlvsink.c