[FFmpeg-devel,3/3] avcodec/videotoolbox: restart decompression session on bad data errors

From: Aman Gupta <aman@tmm1.net>

On some platforms (observed on macOS Sierra with 12" macbook), the VT
decoder will start returning errors when encountering an SPS change in
the h264 bitstream. With this patch, the kVTVideoDecoderBadDataErr
response from the decoder is caught and the decompression session is
recreated with a new avcC. The "bad data" is then fed into the new
decompression session so that it can be decoded correctly.

I discovered the underlying issue here by running ffmpeg with lldb,
which causes macOS to display debug information from the VT hardware
decoder on stderr. The following errors were shown, which indicated the
need to restart the decoder session with a new SPS/avcC:

  ffmpeg[15127:4094995] GVA error: SPS mismatch ...
  ffmpeg[15127:4094995] GVA error: AVF_PushMetaData, first field kAVF_QT0_SPSPPSBoundaryMarker
  ffmpeg[15127:4094995] GVA error: pushMetaData, submitNewJobs
  ffmpeg[15127:4094995] GVA warning: OutputQueueReadyCallback status = 1, buffer == 0x0

Tested with the following sample, which contains an SPS change midstream:
http://tmm1.s3.amazonaws.com/videotoolbox/spschange.ts
---
 libavcodec/videotoolbox.c | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

On Thu, 16 Feb 2017 10:29:38 -0800
Aman Gupta <ffmpeg@tmm1.net> wrote:

> From: Aman Gupta <aman@tmm1.net>
> 
> On some platforms (observed on macOS Sierra with 12" macbook), the VT
> decoder will start returning errors when encountering an SPS change in
> the h264 bitstream. With this patch, the kVTVideoDecoderBadDataErr
> response from the decoder is caught and the decompression session is
> recreated with a new avcC. The "bad data" is then fed into the new
> decompression session so that it can be decoded correctly.
> 
> I discovered the underlying issue here by running ffmpeg with lldb,
> which causes macOS to display debug information from the VT hardware
> decoder on stderr. The following errors were shown, which indicated the
> need to restart the decoder session with a new SPS/avcC:
> 
>   ffmpeg[15127:4094995] GVA error: SPS mismatch ...
>   ffmpeg[15127:4094995] GVA error: AVF_PushMetaData, first field kAVF_QT0_SPSPPSBoundaryMarker
>   ffmpeg[15127:4094995] GVA error: pushMetaData, submitNewJobs
>   ffmpeg[15127:4094995] GVA warning: OutputQueueReadyCallback status = 1, buffer == 0x0
> 
> Tested with the following sample, which contains an SPS change midstream:
> http://tmm1.s3.amazonaws.com/videotoolbox/spschange.ts
> ---
>  libavcodec/videotoolbox.c | 28 ++++++++++++++++++++++++----
>  1 file changed, 24 insertions(+), 4 deletions(-)
> 
> diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
> index 9be7bee..159d98d 100644
> --- a/libavcodec/videotoolbox.c
> +++ b/libavcodec/videotoolbox.c
> @@ -38,6 +38,9 @@
>  
>  #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING  12
>  
> +static void videotoolbox_stop(AVCodecContext *avctx);
> +static int videotoolbox_start(AVCodecContext *avctx);
> +
>  static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
>  {
>      CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
> @@ -350,13 +353,25 @@ static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
>      int status;
>      AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
>      VTContext *vtctx = avctx->internal->hwaccel_priv_data;
> +    int retry;
>  
>      av_buffer_unref(&frame->buf[0]);
>  
>      if (!videotoolbox->session || !vtctx->bitstream)
>          return AVERROR_INVALIDDATA;
>  
> -    status = videotoolbox_session_decode_frame(avctx);
> +    for (retry = 0; retry < 2; retry++) {

Why retry?

> +        status = videotoolbox_session_decode_frame(avctx);
> +
> +        if (status == kVTVideoDecoderBadDataErr) {
> +            av_log(avctx, AV_LOG_DEBUG, "vt decoder got bad data error, restarting..\n");
> +            videotoolbox_stop(avctx);
> +            videotoolbox_start(avctx);

Wouldn't Bad Things happen if the session failed to create for some
reason?

> +            continue;
> +        } else {
> +            break;
> +        }
> +    }
>  
>      if (status) {
>          av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
> @@ -506,7 +521,7 @@ static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecT
>      return cm_fmt_desc;
>  }
>  
> -static int videotoolbox_default_init(AVCodecContext *avctx)
> +static int videotoolbox_start(AVCodecContext *avctx)
>  {
>      AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
>      OSStatus status;
> @@ -587,7 +602,12 @@ static int videotoolbox_default_init(AVCodecContext *avctx)
>      }
>  }
>  
> -static void videotoolbox_default_free(AVCodecContext *avctx)
> +static int videotoolbox_default_init(AVCodecContext *avctx)
> +{
> +    return videotoolbox_start(avctx);
> +}
> +
> +static void videotoolbox_stop(AVCodecContext *avctx)
>  {
>      AVVideotoolboxContext *videotoolbox = avctx->hwaccel_context;
>      if (!videotoolbox)
> @@ -696,7 +716,7 @@ int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *
>  void av_videotoolbox_default_free(AVCodecContext *avctx)
>  {
>  
> -    videotoolbox_default_free(avctx);
> +    videotoolbox_stop(avctx);
>      av_freep(&avctx->hwaccel_context);
>  }
>  #endif /* CONFIG_VIDEOTOOLBOX */