diff mbox series

[FFmpeg-devel,v2,1/4] avcodec/mmaldec: use decoupled dataflow

Message ID 20210924090438.11954-2-cyph1984@gmail.com
State New
Headers show
Series Switch mmaldec to decoupled dataflow
Related show

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_ppc success Make finished
andriy/make_fate_ppc success Make fate finished

Commit Message

Ming Shun Ho Sept. 24, 2021, 9:04 a.m. UTC
MMAL is an fundamentally an asynchronous decoder, which was a bad fit
for the legacy dataflow API. Often multiple packets are enqueued before
a flood of frames are returned from MMAL.

The previous lockstep dataflow meant that any delay in returning packets
from the VPU would cause ctx->queue_decoded_frames to grow with no way
of draining the queue.

Testing this with mpv streaming from a live RTSP source visibly reduced
latency introduced by frames waiting in queue_decoded_frames from
roughly 2s to 0.

Signed-off-by: Ho Ming Shun <cyph1984@gmail.com>
---
 libavcodec/mmaldec.c | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

Comments

Jan Ekström Sept. 24, 2021, 9:30 a.m. UTC | #1
On Fri, Sep 24, 2021 at 12:06 PM Ho Ming Shun <cyph1984@gmail.com> wrote:
>
> MMAL is an fundamentally an asynchronous decoder, which was a bad fit
> for the legacy dataflow API. Often multiple packets are enqueued before
> a flood of frames are returned from MMAL.
>
> The previous lockstep dataflow meant that any delay in returning packets
> from the VPU would cause ctx->queue_decoded_frames to grow with no way
> of draining the queue.
>
> Testing this with mpv streaming from a live RTSP source visibly reduced
> latency introduced by frames waiting in queue_decoded_frames from
> roughly 2s to 0.
>
> Signed-off-by: Ho Ming Shun <cyph1984@gmail.com>

Hi,

Thanks for the interest towards improving the MMAL hwdec. Do you
happen to know if there is a nice docker container or sysroot tarball
available that can be utilized for CI etc purposes to at least verify
that a standard sysroot that is similar to raspbian or so (as that is
the most common setup with raspis, I think?) still works
building-wise?

Jan
Ming Shun Ho Sept. 24, 2021, 10:37 a.m. UTC | #2
On Fri, Sep 24, 2021 at 5:30 PM Jan Ekström <jeebjp@gmail.com> wrote:
>
> On Fri, Sep 24, 2021 at 12:06 PM Ho Ming Shun <cyph1984@gmail.com> wrote:
> >
> > MMAL is an fundamentally an asynchronous decoder, which was a bad fit
> > for the legacy dataflow API. Often multiple packets are enqueued before
> > a flood of frames are returned from MMAL.
> >
> > The previous lockstep dataflow meant that any delay in returning packets
> > from the VPU would cause ctx->queue_decoded_frames to grow with no way
> > of draining the queue.
> >
> > Testing this with mpv streaming from a live RTSP source visibly reduced
> > latency introduced by frames waiting in queue_decoded_frames from
> > roughly 2s to 0.
> >
> > Signed-off-by: Ho Ming Shun <cyph1984@gmail.com>
>
> Hi,
>
> Thanks for the interest towards improving the MMAL hwdec. Do you

MMAL hwdec with MPV happens to be one of the few decent ways to get
h264 1080p video playback on older Rpi (less than 4).

> happen to know if there is a nice docker container or sysroot tarball
> available that can be utilized for CI etc purposes to at least verify
> that a standard sysroot that is similar to raspbian or so (as that is
> the most common setup with raspis, I think?) still works
> building-wise?

Seems like raspbian images on docker are outdated. Personally I grab a
tarball from http://os.archlinuxarm.org/os/ArchLinuxARM-rpi-2-latest.tar.gz
to do my testing. What's differentiates an RPi distro from a more generic
arm image is just the libs and includes in /opt/vc.

>
> Jan
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox series

Patch

diff --git a/libavcodec/mmaldec.c b/libavcodec/mmaldec.c
index 6c3e5d99b6..d4788b33f3 100644
--- a/libavcodec/mmaldec.c
+++ b/libavcodec/mmaldec.c
@@ -578,6 +578,7 @@  static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt,
 
 done:
     av_buffer_unref(&buf);
+    av_packet_unref(avpkt);
     return ret;
 }
 
@@ -663,6 +664,12 @@  static int ffmal_copy_frame(AVCodecContext *avctx,  AVFrame *frame,
                       avctx->pix_fmt, avctx->width, avctx->height);
     }
 
+    frame->sample_aspect_ratio = avctx->sample_aspect_ratio;
+    frame->width = avctx->width;
+    frame->width = avctx->width;
+    frame->height = avctx->height;
+    frame->format = avctx->pix_fmt;
+
     frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts;
     frame->pkt_dts = AV_NOPTS_VALUE;
 
@@ -771,12 +778,12 @@  done:
     return ret;
 }
 
-static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
-                         AVPacket *avpkt)
+static int ffmmal_receive_frame(AVCodecContext *avctx, AVFrame *frame)
 {
     MMALDecodeContext *ctx = avctx->priv_data;
-    AVFrame *frame = data;
+    AVPacket avpkt = {0};
     int ret = 0;
+    int got_frame = 0;
 
     if (avctx->extradata_size && !ctx->extradata_sent) {
         AVPacket pkt = {0};
@@ -788,7 +795,11 @@  static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
             return ret;
     }
 
-    if ((ret = ffmmal_add_packet(avctx, avpkt, 0)) < 0)
+    ret = ff_decode_get_packet(avctx, &avpkt);
+    if(ret == 0) {
+        if ((ret = ffmmal_add_packet(avctx, &avpkt, 0)) < 0)
+            return ret;
+    } else if(ret < 0 && !(ret == AVERROR(EAGAIN)))
         return ret;
 
     if ((ret = ffmmal_fill_input_port(avctx)) < 0)
@@ -797,7 +808,7 @@  static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
     if ((ret = ffmmal_fill_output_port(avctx)) < 0)
         return ret;
 
-    if ((ret = ffmmal_read_frame(avctx, frame, got_frame)) < 0)
+    if ((ret = ffmmal_read_frame(avctx, frame, &got_frame)) < 0)
         return ret;
 
     // ffmmal_read_frame() can block for a while. Since the decoder is
@@ -809,7 +820,10 @@  static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
     if ((ret = ffmmal_fill_input_port(avctx)) < 0)
         return ret;
 
-    return ret;
+    if(!got_frame && ret == 0)
+        return AVERROR(EAGAIN);
+    else
+        return ret;
 }
 
 static const AVCodecHWConfigInternal *const mmal_hw_configs[] = {
@@ -841,7 +855,7 @@  static const AVOption options[]={
         .priv_data_size = sizeof(MMALDecodeContext), \
         .init           = ffmmal_init_decoder, \
         .close          = ffmmal_close_decoder, \
-        .decode         = ffmmal_decode, \
+        .receive_frame  = ffmmal_receive_frame, \
         .flush          = ffmmal_flush, \
         .priv_class     = &ffmmal_##NAME##_dec_class, \
         .capabilities   = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, \