diff mbox series

[FFmpeg-devel,v3,4/4] lavc, doc: add libuavs3d video decoder wrapper

Message ID 20200805161831.446-5-hwrenx@126.com
State Superseded
Headers show
Series Supplement AVS3-P2/IEEE1857.10 video decoding via libuavs3d
Related show

Checks

Context Check Description
andriy/default pending
andriy/configure warning Failed to apply patch

Commit Message

hwren Aug. 5, 2020, 4:18 p.m. UTC
From: hwren <hwrenx@126.com>

Signed-off-by: hbj <hanbj@pku.edu.cn>
Signed-off-by: hwren <hwrenx@126.com>
---
 Changelog              |   1 +
 configure              |   4 +
 doc/decoders.texi      |  21 +++
 doc/general.texi       |   8 ++
 libavcodec/Makefile    |   1 +
 libavcodec/allcodecs.c |   1 +
 libavcodec/libuavs3d.c | 283 +++++++++++++++++++++++++++++++++++++++++
 7 files changed, 319 insertions(+)
 create mode 100644 libavcodec/libuavs3d.c

Comments

James Almer Aug. 5, 2020, 9:21 p.m. UTC | #1
On 8/5/2020 1:18 PM, hwrenx@126.com wrote:
> From: hwren <hwrenx@126.com>
> 
> Signed-off-by: hbj <hanbj@pku.edu.cn>
> Signed-off-by: hwren <hwrenx@126.com>
> ---
>  Changelog              |   1 +
>  configure              |   4 +
>  doc/decoders.texi      |  21 +++
>  doc/general.texi       |   8 ++
>  libavcodec/Makefile    |   1 +
>  libavcodec/allcodecs.c |   1 +
>  libavcodec/libuavs3d.c | 283 +++++++++++++++++++++++++++++++++++++++++
>  7 files changed, 319 insertions(+)
>  create mode 100644 libavcodec/libuavs3d.c
> 
> diff --git a/Changelog b/Changelog
> index a60e7d2eb8..dfd56b3fc6 100644
> --- a/Changelog
> +++ b/Changelog
> @@ -4,6 +4,7 @@ releases are sorted from youngest to oldest.
>  version <next>:
>  - AudioToolbox output device
>  - MacCaption demuxer
> +- AVS3 video decoder via libuavs3d
>  
>  
>  version 4.3:
> diff --git a/configure b/configure
> index 7495f35faa..7340bc4a35 100755
> --- a/configure
> +++ b/configure
> @@ -274,6 +274,7 @@ External library support:
>    --enable-libtls          enable LibreSSL (via libtls), needed for https support
>                             if openssl, gnutls or mbedtls is not used [no]
>    --enable-libtwolame      enable MP2 encoding via libtwolame [no]
> +  --enable-libuavs3d       enable AVS3 decoding via libuavs3d [no]
>    --enable-libv4l2         enable libv4l2/v4l-utils [no]
>    --enable-libvidstab      enable video stabilization using vid.stab [no]
>    --enable-libvmaf         enable vmaf filter via libvmaf [no]
> @@ -1807,6 +1808,7 @@ EXTERNAL_LIBRARY_LIST="
>      libtesseract
>      libtheora
>      libtwolame
> +    libuavs3d
>      libv4l2
>      libvorbis
>      libvpx
> @@ -3242,6 +3244,7 @@ libspeex_encoder_deps="libspeex"
>  libspeex_encoder_select="audio_frame_queue"
>  libtheora_encoder_deps="libtheora"
>  libtwolame_encoder_deps="libtwolame"
> +libuavs3d_decoder_deps="libuavs3d"
>  libvo_amrwbenc_encoder_deps="libvo_amrwbenc"
>  libvorbis_decoder_deps="libvorbis"
>  libvorbis_encoder_deps="libvorbis libvorbisenc"
> @@ -6379,6 +6382,7 @@ enabled libtls            && require_pkg_config libtls libtls tls.h tls_configur
>  enabled libtwolame        && require libtwolame twolame.h twolame_init -ltwolame &&
>                               { check_lib libtwolame twolame.h twolame_encode_buffer_float32_interleaved -ltwolame ||
>                                 die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; }
> +enabled libuavs3d         && require_pkg_config libuavs3d uavs3d uavs3d.h uavs3d_decode
>  enabled libv4l2           && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_ioctl
>  enabled libvidstab        && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit
>  enabled libvmaf           && require_pkg_config libvmaf "libvmaf >= 1.3.9" libvmaf.h compute_vmaf
> diff --git a/doc/decoders.texi b/doc/decoders.texi
> index 9005714e3c..f1a0b3c36e 100644
> --- a/doc/decoders.texi
> +++ b/doc/decoders.texi
> @@ -86,6 +86,27 @@ AVS2-P2/IEEE1857.4 video decoder wrapper.
>  
>  This decoder allows libavcodec to decode AVS2 streams with davs2 library.
>  
> +@c man end VIDEO DECODERS
> + 
> +@section libuavs3d
> +
> +AVS3-P2/IEEE1857.10 video decoder.
> +
> +libuavs3d allows libavcodec to decode AVS3 streams.
> +Requires the presence of the libuavs3d headers and library during configuration.
> +You need to explicitly configure the build with @code{--enable-libuavs3d}.
> +
> +@subsection Options
> +
> +The following option is supported by the libuavs3d wrapper.
> +
> +@table @option
> +
> +@item frame_threads
> +Set amount of frame threads to use during decoding. The default value is 0 (autodetect).
> +
> +@end table
> +
>  @c man end VIDEO DECODERS
>  
>  @chapter Audio Decoders
> diff --git a/doc/general.texi b/doc/general.texi
> index 9b0ee96752..6d673b74e1 100644
> --- a/doc/general.texi
> +++ b/doc/general.texi
> @@ -125,6 +125,14 @@ Go to @url{https://github.com/pkuvcl/davs2} and follow the instructions for
>  installing the library. Then pass @code{--enable-libdavs2} to configure to
>  enable it.
>  
> +@section uavs3d
> +
> +FFmpeg can make use of the uavs3d library for AVS3-P2/IEEE1857.10 video decoding.
> +
> +Go to @url{https://github.com/uavs3/uavs3d} and follow the instructions for
> +installing the library. Then pass @code{--enable-libuavs3d} to configure to
> +enable it.
> +
>  @float NOTE
>  libdavs2 is under the GNU Public License Version 2 or later
>  (see @url{http://www.gnu.org/licenses/old-licenses/gpl-2.0.html} for
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index f1512779be..491485f3c0 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -1026,6 +1026,7 @@ OBJS-$(CONFIG_LIBSPEEX_DECODER)           += libspeexdec.o
>  OBJS-$(CONFIG_LIBSPEEX_ENCODER)           += libspeexenc.o
>  OBJS-$(CONFIG_LIBTHEORA_ENCODER)          += libtheoraenc.o
>  OBJS-$(CONFIG_LIBTWOLAME_ENCODER)         += libtwolame.o
> +OBJS-$(CONFIG_LIBUAVS3D_DECODER)          += libuavs3d.o
>  OBJS-$(CONFIG_LIBVO_AMRWBENC_ENCODER)     += libvo-amrwbenc.o
>  OBJS-$(CONFIG_LIBVORBIS_DECODER)          += libvorbisdec.o
>  OBJS-$(CONFIG_LIBVORBIS_ENCODER)          += libvorbisenc.o \
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index 80f128cade..3d2d0af87a 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -730,6 +730,7 @@ extern AVCodec ff_libspeex_encoder;
>  extern AVCodec ff_libspeex_decoder;
>  extern AVCodec ff_libtheora_encoder;
>  extern AVCodec ff_libtwolame_encoder;
> +extern AVCodec ff_libuavs3d_decoder;
>  extern AVCodec ff_libvo_amrwbenc_encoder;
>  extern AVCodec ff_libvorbis_encoder;
>  extern AVCodec ff_libvorbis_decoder;
> diff --git a/libavcodec/libuavs3d.c b/libavcodec/libuavs3d.c
> new file mode 100644
> index 0000000000..c0d89cd1ce
> --- /dev/null
> +++ b/libavcodec/libuavs3d.c
> @@ -0,0 +1,283 @@
> +/*
> + * RAW AVS3-P2/IEEE1857.10 video demuxer
> + * Copyright (c) 2020 Zhenyu Wang <wangzhenyu@pkusz.edu.cn>
> + *                    Bingjie Han <hanbj@pkusz.edu.cn>
> + *                    Huiwen Ren  <hwrenx@gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/avutil.h"
> +#include "libavutil/common.h"
> +#include "libavutil/imgutils.h"
> +#include "libavutil/opt.h"
> +#include "avcodec.h"
> +#include "internal.h"
> +#include "uavs3d.h"
> +
> +#define UAVS3D_MAX_FRAME_THREADS 48 
> +
> +static const int color_primaries_tab[10] = {
> +    AVCOL_PRI_RESERVED0   ,    // 0
> +    AVCOL_PRI_BT709       ,    // 1
> +    AVCOL_PRI_UNSPECIFIED ,    // 2
> +    AVCOL_PRI_RESERVED    ,    // 3
> +    AVCOL_PRI_BT470M      ,    // 4
> +    AVCOL_PRI_BT470BG      ,    // 5
> +    AVCOL_PRI_SMPTE170M   ,    // 6
> +    AVCOL_PRI_SMPTE240M   ,    // 7
> +    AVCOL_PRI_FILM        ,    // 8
> +    AVCOL_PRI_BT2020           // 9  
> +};
> +    
> +static const int color_transfer_tab[15] = {
> +    AVCOL_TRC_RESERVED0    , // 0
> +    AVCOL_TRC_BT709        , // 1
> +    AVCOL_TRC_UNSPECIFIED  , // 2
> +    AVCOL_TRC_RESERVED     , // 3
> +    AVCOL_TRC_GAMMA22      , // 4
> +    AVCOL_TRC_GAMMA28      , // 5
> +    AVCOL_TRC_SMPTE170M    , // 6
> +    AVCOL_TRC_SMPTE240M    , // 7
> +    AVCOL_TRC_LINEAR       , // 8
> +    AVCOL_TRC_LOG          , // 9
> +    AVCOL_TRC_LOG_SQRT     , // 10
> +    AVCOL_TRC_BT2020_12    , // 11
> +    AVCOL_TRC_SMPTE2084    , // 12
> +    AVCOL_TRC_UNSPECIFIED  , // 13
> +    AVCOL_TRC_ARIB_STD_B67   // 14
> +};
> +
> +static const int color_matrix_tab[12] = {
> +    AVCOL_SPC_RESERVED     , // 0
> +    AVCOL_SPC_BT709        , // 1
> +    AVCOL_SPC_UNSPECIFIED  , // 2
> +    AVCOL_SPC_RESERVED     , // 3
> +    AVCOL_SPC_FCC          , // 4
> +    AVCOL_SPC_BT470BG      , // 5
> +    AVCOL_SPC_SMPTE170M    , // 6
> +    AVCOL_SPC_SMPTE240M    , // 7
> +    AVCOL_SPC_BT2020_NCL   , // 8
> +    AVCOL_SPC_BT2020_CL    , // 9
> +    AVCOL_SPC_UNSPECIFIED  , // 10
> +    AVCOL_SPC_UNSPECIFIED    // 11
> +};
> +
> +static const enum AVPictureType IMGTYPE[8] = {
> +    AV_PICTURE_TYPE_NONE,
> +    AV_PICTURE_TYPE_I,
> +    AV_PICTURE_TYPE_P,
> +    AV_PICTURE_TYPE_B
> +};
> +
> +typedef struct UAVS3DContext {
> +    AVCodecContext  *avctx;
> +    void            *dec_handle;
> +    int              frame_threads;
> +    int              got_seqhdr;
> +    uavs3d_io_frm_t  dec_frame;
> +} UAVS3DContext;
> +
> +
> +static int uavs3d_find_next_start_code(const unsigned char *bs_data, int bs_len, int *left)
> +{
> +    const unsigned char *data_ptr = bs_data + 4;
> +    int count = bs_len - 4;
> +
> +    while (count >= 4 &&
> +        ((*(unsigned int *)data_ptr) != 0xB6010000) && /* P/B picture */
> +        ((*(unsigned int *)data_ptr) != 0xB3010000) && /* I   picture */
> +        ((*(unsigned int *)data_ptr) != 0xB0010000) && /* sequence header */
> +        ((*(unsigned int *)data_ptr) != 0x00010000) && /* first slice */
> +        ((*(unsigned int *)data_ptr) != 0xB1010000)) { /* sequence end */
> +        data_ptr++;
> +        count--;
> +    }
> +
> +    if (count >= 4) {
> +        *left = count; 
> +        return 1;
> +    }
> +
> +    return 0;
> +}
> +
> +static void uavs3d_output_callback(uavs3d_io_frm_t *dec_frame) {
> +    uavs3d_io_frm_t frm_out;
> +    AVFrame *frm = (AVFrame *)dec_frame->priv;
> +    int i;
> +
> +    if (!frm) {
> +        return;
> +    }
> +    
> +    frm->pts       = dec_frame->pts;
> +    frm->pkt_dts   = dec_frame->dts;
> +    frm->pict_type = IMGTYPE[dec_frame->type];
> +    frm->key_frame = (frm->pict_type == AV_PICTURE_TYPE_I);
> +
> +    for (i = 0; i < 3; i++) {
> +        frm_out.width [i] = dec_frame->width[i];
> +        frm_out.height[i] = dec_frame->height[i];
> +        frm_out.stride[i] = frm->linesize[i];
> +        frm_out.buffer[i] = frm->data[i]; 
> +    }
> +
> +    uavs3d_img_cpy_cvt(&frm_out, dec_frame, dec_frame->bit_depth);
> +}
> +
> +static av_cold int libuavs3d_init(AVCodecContext *avctx)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +    uavs3d_cfg_t cdsc;
> +
> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
> +    cdsc.check_md5 = 0;
> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
> +    h->got_seqhdr = 0;
> + 
> +    return 0;
> +}
> +
> +static av_cold int libuavs3d_end(AVCodecContext *avctx)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +    
> +    if (h->dec_handle) {
> +        uavs3d_flush(h->dec_handle, NULL);
> +        uavs3d_delete(h->dec_handle);
> +        h->dec_handle = NULL;
> +    }
> +    h->got_seqhdr = 0;
> +    
> +    return 0;
> +}
> +
> +static void libuavs3d_flush(AVCodecContext * avctx)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +    uavs3d_cfg_t cdsc;
> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
> +    cdsc.check_md5 = 0;
> +
> +    if (h->dec_handle) {
> +        uavs3d_flush(h->dec_handle, NULL);
> +        uavs3d_delete(h->dec_handle);
> +    }
> +     
> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);

Is it really necessary to destroy and create the decoder context on
every seek? It feels slow and inefficient.

> +    h->got_seqhdr = 0;
> +}
> +
> +static int libuavs3d_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
> +{

Looking at this function, it seems that this decoder would be better
implemented using the decoupled input/output AVCodec.receive_frame API
instead of the AVCodec.decode one like you're doing here, as it lets you
request for packets as you need them.

See binkaudio and libdav1d decoders for example implementations: You
fetch one packet, keep it around and return frames until it's fully
consumed, then attempt to fetch another and repeat the process.

> +    UAVS3DContext *h = avctx->priv_data;
> +    const uint8_t *buf = avpkt->data;
> +    int buf_size = avpkt->size;
> +    const uint8_t *buf_end;
> +    const uint8_t *buf_ptr;
> +    AVFrame *frm = (AVFrame*)data;
> +    int left_bytes;
> +    int ret, finish = 0;
> +
> +    *got_frame = 0;
> +    frm->pts = -1;
> +    frm->pict_type = AV_PICTURE_TYPE_NONE;
> +
> +    if (h->got_seqhdr) {
> +        if (!frm->data[0] && (ret = ff_get_buffer(avctx, frm, 0)) < 0) {
> +            return ret;
> +        } 
> +        h->dec_frame.priv = data;   // AVFrame
> +    }
> +
> +    if (!buf_size) {
> +        do {
> +            ret = uavs3d_flush(h->dec_handle, &h->dec_frame);
> +        } while (ret > 0 && !h->dec_frame.got_pic);
> +    } else {
> +        buf_ptr = buf;
> +        buf_end = buf + buf_size;
> +  
> +        while (!finish) {
> +            int bs_len;
> +            uavs3d_io_frm_t *frm_dec = &h->dec_frame;
> +
> +            if (uavs3d_find_next_start_code(buf_ptr, buf_end - buf_ptr, &left_bytes)) {
> +                bs_len = buf_end - buf_ptr - left_bytes;
> +            } else { 
> +                bs_len = buf_end - buf_ptr;
> +                finish = 1;
> +            }
> +            frm_dec->bs = (unsigned char *)buf_ptr;
> +            frm_dec->bs_len = bs_len;
> +            frm_dec->pts = avpkt->pts;
> +            frm_dec->dts = avpkt->dts;
> +            uavs3d_decode(h->dec_handle, frm_dec);
> +            buf_ptr += bs_len;
> +
> +            if (frm_dec->nal_type == NAL_SEQ_HEADER) {
> +                static const int avs3_fps_num[9] = {0, 240000, 24, 25, 30000, 30, 50, 60000, 60 };
> +                static const int avs3_fps_den[9] = {1,   1001,  1,  1,  1001,  1,  1,  1001,  1 };
> +                avctx->framerate.num = avs3_fps_num[frm_dec->seqhdr->frame_rate_code];
> +                avctx->framerate.den = avs3_fps_den[frm_dec->seqhdr->frame_rate_code];
> +                avctx->has_b_frames = 1;
> +                avctx->pix_fmt = frm_dec->seqhdr->bit_depth_internal == 8 ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_YUV420P10LE;
> +                ff_set_dimensions(avctx, frm_dec->seqhdr->horizontal_size, frm_dec->seqhdr->vertical_size);
> +                h->got_seqhdr = 1;
> +            }
> +            if (frm_dec->got_pic) {
> +                break;
> +            }
> +        }
> +    }
> +
> +    *got_frame = h->dec_frame.got_pic;
> +
> +    return buf_ptr - buf;
> +}
> +
> +#define UAVS3D_OFFSET(x) offsetof(UAVS3DContext, x)
> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
> +static const AVOption options[] = {
> +    { "frame_threads",      "number of frame-level threads ", UAVS3D_OFFSET(frame_threads),  AV_OPT_TYPE_INT,    {.i64 =  0 }, 0, UAVS3D_MAX_FRAME_THREADS, VE },

No need for a custom option and UAVS3DContext field, you have
avctx->threads for this.
Just add the AV_CODEC_CAP_AUTO_THREADS capability to
ff_libuavs3d_decoder and it will default to 0.

> +    { NULL }
> +};
> +static const AVClass libuavs3d_class = {
> +    .class_name = "libuavs3d_class",
> +    .item_name  = av_default_item_name,
> +    .option     = options,
> +    .version    = LIBAVUTIL_VERSION_INT,
> +};
> +
> +AVCodec ff_libuavs3d_decoder = {
> +    .name           = "libuavs3d",
> +    .long_name      = NULL_IF_CONFIG_SMALL("uavs3d AVS3-P2/IEEE1857.10 decoder"),
> +    .type           = AVMEDIA_TYPE_VIDEO,
> +    .id             = AV_CODEC_ID_AVS3,
> +    .priv_data_size = sizeof(UAVS3DContext),
> +    .priv_class     = &libuavs3d_class,
> +    .init           = libuavs3d_init,
> +    .close          = libuavs3d_end,
> +    .decode         = libuavs3d_decode_frame,
> +    .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY,
> +    .flush          = libuavs3d_flush,
> +    .pix_fmts        = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_NONE },
> +    
> +    .wrapper_name   = "libuavs3d",
> +};
>
hwren Aug. 8, 2020, 5 p.m. UTC | #2
At 2020-08-06 05:21:43, "James Almer" <jamrial@gmail.com> wrote:
>On 8/5/2020 1:18 PM, hwrenx@126.com wrote:
>> From: hwren <hwrenx@126.com>
>> 
>> Signed-off-by: hbj <hanbj@pku.edu.cn>
>> Signed-off-by: hwren <hwrenx@126.com>
>> ---
>>  Changelog              |   1 +
>>  configure              |   4 +
>>  doc/decoders.texi      |  21 +++
>>  doc/general.texi       |   8 ++
>>  libavcodec/Makefile    |   1 +
>>  libavcodec/allcodecs.c |   1 +
>>  libavcodec/libuavs3d.c | 283 +++++++++++++++++++++++++++++++++++++++++
>>  7 files changed, 319 insertions(+)
>>  create mode 100644 libavcodec/libuavs3d.c
>> 
>> diff --git a/Changelog b/Changelog
>> index a60e7d2eb8..dfd56b3fc6 100644
>> --- a/Changelog
>> +++ b/Changelog
>> @@ -4,6 +4,7 @@ releases are sorted from youngest to oldest.
>>  version <next>:
>>  - AudioToolbox output device
>>  - MacCaption demuxer
>> +- AVS3 video decoder via libuavs3d
>>  
>>  
>>  version 4.3:
>> diff --git a/configure b/configure
>> index 7495f35faa..7340bc4a35 100755
>> --- a/configure
>> +++ b/configure
>> @@ -274,6 +274,7 @@ External library support:
>>    --enable-libtls          enable LibreSSL (via libtls), needed for https support
>>                             if openssl, gnutls or mbedtls is not used [no]
>>    --enable-libtwolame      enable MP2 encoding via libtwolame [no]
>> +  --enable-libuavs3d       enable AVS3 decoding via libuavs3d [no]
>>    --enable-libv4l2         enable libv4l2/v4l-utils [no]
>>    --enable-libvidstab      enable video stabilization using vid.stab [no]
>>    --enable-libvmaf         enable vmaf filter via libvmaf [no]
>> @@ -1807,6 +1808,7 @@ EXTERNAL_LIBRARY_LIST="
>>      libtesseract
>>      libtheora
>>      libtwolame
>> +    libuavs3d
>>      libv4l2
>>      libvorbis
>>      libvpx
>> @@ -3242,6 +3244,7 @@ libspeex_encoder_deps="libspeex"
>>  libspeex_encoder_select="audio_frame_queue"
>>  libtheora_encoder_deps="libtheora"
>>  libtwolame_encoder_deps="libtwolame"
>> +libuavs3d_decoder_deps="libuavs3d"
>>  libvo_amrwbenc_encoder_deps="libvo_amrwbenc"
>>  libvorbis_decoder_deps="libvorbis"
>>  libvorbis_encoder_deps="libvorbis libvorbisenc"
>> @@ -6379,6 +6382,7 @@ enabled libtls            && require_pkg_config libtls libtls tls.h tls_configur
>>  enabled libtwolame        && require libtwolame twolame.h twolame_init -ltwolame &&
>>                               { check_lib libtwolame twolame.h twolame_encode_buffer_float32_interleaved -ltwolame ||
>>                                 die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; }
>> +enabled libuavs3d         && require_pkg_config libuavs3d uavs3d uavs3d.h uavs3d_decode
>>  enabled libv4l2           && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_ioctl
>>  enabled libvidstab        && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit
>>  enabled libvmaf           && require_pkg_config libvmaf "libvmaf >= 1.3.9" libvmaf.h compute_vmaf
>> diff --git a/doc/decoders.texi b/doc/decoders.texi
>> index 9005714e3c..f1a0b3c36e 100644
>> --- a/doc/decoders.texi
>> +++ b/doc/decoders.texi
>> @@ -86,6 +86,27 @@ AVS2-P2/IEEE1857.4 video decoder wrapper.
>>  
>>  This decoder allows libavcodec to decode AVS2 streams with davs2 library.
>>  
>> +@c man end VIDEO DECODERS
>> + 
>> +@section libuavs3d
>> +
>> +AVS3-P2/IEEE1857.10 video decoder.
>> +
>> +libuavs3d allows libavcodec to decode AVS3 streams.
>> +Requires the presence of the libuavs3d headers and library during configuration.
>> +You need to explicitly configure the build with @code{--enable-libuavs3d}.
>> +
>> +@subsection Options
>> +
>> +The following option is supported by the libuavs3d wrapper.
>> +
>> +@table @option
>> +
>> +@item frame_threads
>> +Set amount of frame threads to use during decoding. The default value is 0 (autodetect).
>> +
>> +@end table
>> +
>>  @c man end VIDEO DECODERS
>>  
>>  @chapter Audio Decoders
>> diff --git a/doc/general.texi b/doc/general.texi
>> index 9b0ee96752..6d673b74e1 100644
>> --- a/doc/general.texi
>> +++ b/doc/general.texi
>> @@ -125,6 +125,14 @@ Go to @url{https://github.com/pkuvcl/davs2} and follow the instructions for
>>  installing the library. Then pass @code{--enable-libdavs2} to configure to
>>  enable it.
>>  
>> +@section uavs3d
>> +
>> +FFmpeg can make use of the uavs3d library for AVS3-P2/IEEE1857.10 video decoding.
>> +
>> +Go to @url{https://github.com/uavs3/uavs3d} and follow the instructions for
>> +installing the library. Then pass @code{--enable-libuavs3d} to configure to
>> +enable it.
>> +
>>  @float NOTE
>>  libdavs2 is under the GNU Public License Version 2 or later
>>  (see @url{http://www.gnu.org/licenses/old-licenses/gpl-2.0.html} for
>> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
>> index f1512779be..491485f3c0 100644
>> --- a/libavcodec/Makefile
>> +++ b/libavcodec/Makefile
>> @@ -1026,6 +1026,7 @@ OBJS-$(CONFIG_LIBSPEEX_DECODER)           += libspeexdec.o
>>  OBJS-$(CONFIG_LIBSPEEX_ENCODER)           += libspeexenc.o
>>  OBJS-$(CONFIG_LIBTHEORA_ENCODER)          += libtheoraenc.o
>>  OBJS-$(CONFIG_LIBTWOLAME_ENCODER)         += libtwolame.o
>> +OBJS-$(CONFIG_LIBUAVS3D_DECODER)          += libuavs3d.o
>>  OBJS-$(CONFIG_LIBVO_AMRWBENC_ENCODER)     += libvo-amrwbenc.o
>>  OBJS-$(CONFIG_LIBVORBIS_DECODER)          += libvorbisdec.o
>>  OBJS-$(CONFIG_LIBVORBIS_ENCODER)          += libvorbisenc.o \
>> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
>> index 80f128cade..3d2d0af87a 100644
>> --- a/libavcodec/allcodecs.c
>> +++ b/libavcodec/allcodecs.c
>> @@ -730,6 +730,7 @@ extern AVCodec ff_libspeex_encoder;
>>  extern AVCodec ff_libspeex_decoder;
>>  extern AVCodec ff_libtheora_encoder;
>>  extern AVCodec ff_libtwolame_encoder;
>> +extern AVCodec ff_libuavs3d_decoder;
>>  extern AVCodec ff_libvo_amrwbenc_encoder;
>>  extern AVCodec ff_libvorbis_encoder;
>>  extern AVCodec ff_libvorbis_decoder;
>> diff --git a/libavcodec/libuavs3d.c b/libavcodec/libuavs3d.c
>> new file mode 100644
>> index 0000000000..c0d89cd1ce
>> --- /dev/null
>> +++ b/libavcodec/libuavs3d.c
>> @@ -0,0 +1,283 @@
>> +/*
>> + * RAW AVS3-P2/IEEE1857.10 video demuxer
>> + * Copyright (c) 2020 Zhenyu Wang <wangzhenyu@pkusz.edu.cn>
>> + *                    Bingjie Han <hanbj@pkusz.edu.cn>
>> + *                    Huiwen Ren  <hwrenx@gmail.com>
>> + *
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>> + */
>> +
>> +#include "libavutil/avassert.h"
>> +#include "libavutil/avutil.h"
>> +#include "libavutil/common.h"
>> +#include "libavutil/imgutils.h"
>> +#include "libavutil/opt.h"
>> +#include "avcodec.h"
>> +#include "internal.h"
>> +#include "uavs3d.h"
>> +
>> +#define UAVS3D_MAX_FRAME_THREADS 48 
>> +
>> +static const int color_primaries_tab[10] = {
>> +    AVCOL_PRI_RESERVED0   ,    // 0
>> +    AVCOL_PRI_BT709       ,    // 1
>> +    AVCOL_PRI_UNSPECIFIED ,    // 2
>> +    AVCOL_PRI_RESERVED    ,    // 3
>> +    AVCOL_PRI_BT470M      ,    // 4
>> +    AVCOL_PRI_BT470BG      ,    // 5
>> +    AVCOL_PRI_SMPTE170M   ,    // 6
>> +    AVCOL_PRI_SMPTE240M   ,    // 7
>> +    AVCOL_PRI_FILM        ,    // 8
>> +    AVCOL_PRI_BT2020           // 9  
>> +};
>> +    
>> +static const int color_transfer_tab[15] = {
>> +    AVCOL_TRC_RESERVED0    , // 0
>> +    AVCOL_TRC_BT709        , // 1
>> +    AVCOL_TRC_UNSPECIFIED  , // 2
>> +    AVCOL_TRC_RESERVED     , // 3
>> +    AVCOL_TRC_GAMMA22      , // 4
>> +    AVCOL_TRC_GAMMA28      , // 5
>> +    AVCOL_TRC_SMPTE170M    , // 6
>> +    AVCOL_TRC_SMPTE240M    , // 7
>> +    AVCOL_TRC_LINEAR       , // 8
>> +    AVCOL_TRC_LOG          , // 9
>> +    AVCOL_TRC_LOG_SQRT     , // 10
>> +    AVCOL_TRC_BT2020_12    , // 11
>> +    AVCOL_TRC_SMPTE2084    , // 12
>> +    AVCOL_TRC_UNSPECIFIED  , // 13
>> +    AVCOL_TRC_ARIB_STD_B67   // 14
>> +};
>> +
>> +static const int color_matrix_tab[12] = {
>> +    AVCOL_SPC_RESERVED     , // 0
>> +    AVCOL_SPC_BT709        , // 1
>> +    AVCOL_SPC_UNSPECIFIED  , // 2
>> +    AVCOL_SPC_RESERVED     , // 3
>> +    AVCOL_SPC_FCC          , // 4
>> +    AVCOL_SPC_BT470BG      , // 5
>> +    AVCOL_SPC_SMPTE170M    , // 6
>> +    AVCOL_SPC_SMPTE240M    , // 7
>> +    AVCOL_SPC_BT2020_NCL   , // 8
>> +    AVCOL_SPC_BT2020_CL    , // 9
>> +    AVCOL_SPC_UNSPECIFIED  , // 10
>> +    AVCOL_SPC_UNSPECIFIED    // 11
>> +};
>> +
>> +static const enum AVPictureType IMGTYPE[8] = {
>> +    AV_PICTURE_TYPE_NONE,
>> +    AV_PICTURE_TYPE_I,
>> +    AV_PICTURE_TYPE_P,
>> +    AV_PICTURE_TYPE_B
>> +};
>> +
>> +typedef struct UAVS3DContext {
>> +    AVCodecContext  *avctx;
>> +    void            *dec_handle;
>> +    int              frame_threads;
>> +    int              got_seqhdr;
>> +    uavs3d_io_frm_t  dec_frame;
>> +} UAVS3DContext;
>> +
>> +
>> +static int uavs3d_find_next_start_code(const unsigned char *bs_data, int bs_len, int *left)
>> +{
>> +    const unsigned char *data_ptr = bs_data + 4;
>> +    int count = bs_len - 4;
>> +
>> +    while (count >= 4 &&
>> +        ((*(unsigned int *)data_ptr) != 0xB6010000) && /* P/B picture */
>> +        ((*(unsigned int *)data_ptr) != 0xB3010000) && /* I   picture */
>> +        ((*(unsigned int *)data_ptr) != 0xB0010000) && /* sequence header */
>> +        ((*(unsigned int *)data_ptr) != 0x00010000) && /* first slice */
>> +        ((*(unsigned int *)data_ptr) != 0xB1010000)) { /* sequence end */
>> +        data_ptr++;
>> +        count--;
>> +    }
>> +
>> +    if (count >= 4) {
>> +        *left = count; 
>> +        return 1;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +static void uavs3d_output_callback(uavs3d_io_frm_t *dec_frame) {
>> +    uavs3d_io_frm_t frm_out;
>> +    AVFrame *frm = (AVFrame *)dec_frame->priv;
>> +    int i;
>> +
>> +    if (!frm) {
>> +        return;
>> +    }
>> +    
>> +    frm->pts       = dec_frame->pts;
>> +    frm->pkt_dts   = dec_frame->dts;
>> +    frm->pict_type = IMGTYPE[dec_frame->type];
>> +    frm->key_frame = (frm->pict_type == AV_PICTURE_TYPE_I);
>> +
>> +    for (i = 0; i < 3; i++) {
>> +        frm_out.width [i] = dec_frame->width[i];
>> +        frm_out.height[i] = dec_frame->height[i];
>> +        frm_out.stride[i] = frm->linesize[i];
>> +        frm_out.buffer[i] = frm->data[i]; 
>> +    }
>> +
>> +    uavs3d_img_cpy_cvt(&frm_out, dec_frame, dec_frame->bit_depth);
>> +}
>> +
>> +static av_cold int libuavs3d_init(AVCodecContext *avctx)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +    uavs3d_cfg_t cdsc;
>> +
>> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
>> +    cdsc.check_md5 = 0;
>> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
>> +    h->got_seqhdr = 0;
>> + 
>> +    return 0;
>> +}
>> +
>> +static av_cold int libuavs3d_end(AVCodecContext *avctx)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +    
>> +    if (h->dec_handle) {
>> +        uavs3d_flush(h->dec_handle, NULL);
>> +        uavs3d_delete(h->dec_handle);
>> +        h->dec_handle = NULL;
>> +    }
>> +    h->got_seqhdr = 0;
>> +    
>> +    return 0;
>> +}
>> +
>> +static void libuavs3d_flush(AVCodecContext * avctx)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +    uavs3d_cfg_t cdsc;
>> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
>> +    cdsc.check_md5 = 0;
>> +
>> +    if (h->dec_handle) {
>> +        uavs3d_flush(h->dec_handle, NULL);
>> +        uavs3d_delete(h->dec_handle);
>> +    }
>> +     
>> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
>
>Is it really necessary to destroy and create the decoder context on
>every seek? It feels slow and inefficient.

No, it is redundant. Will be fixed it in the next version. Thanks for pointing out.

>
>> +    h->got_seqhdr = 0;
>> +}
>> +
>> +static int libuavs3d_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
>> +{
>
>Looking at this function, it seems that this decoder would be better
>implemented using the decoupled input/output AVCodec.receive_frame API
>instead of the AVCodec.decode one like you're doing here, as it lets you
>request for packets as you need them.
>
>See binkaudio and libdav1d decoders for example implementations: You
>fetch one packet, keep it around and return frames until it's fully
>consumed, then attempt to fetch another and repeat the process.
>

I am not sure if I correctly understand the difference between AVCodec.receive_frame
and AVCodec.decode. It seems that both AVCodec.receive_frame and AVCodec.decode
are used in function decode_receive_frame_internal in lavc/decode.c.

Once we use the AVCodec.decode, we have to wait until the frame buffer is filled,
(which means we got a frame). If we use the AVCodec.receive_frame, we fetch one packet,
send it to decoder directly, then request output from the decoder. Considering that the
decode_receive_frame function should get one decoded frame data in each receive call,
and not all packets in AVS3 contain frame data, these two functions would be equivalent
in some respects.

I mean, both of them fetch and send packet, and wait for output. Cause AVS3 packets
may contain 0 or 1 frame data, the libuavs3d_receive_frame function still needs to wait,
and may be exactly the same format as decode_simple_receive_frame, which is the upper
encapsulation of AVCodec.decode in lavc/decode.c.

So...I haven't got how to optimize this part or receive_frame should be used in which
condition. Please tell me more and correct me if there is any problem. Thanks.

>> +    UAVS3DContext *h = avctx->priv_data;
>> +    const uint8_t *buf = avpkt->data;
>> +    int buf_size = avpkt->size;
>> +    const uint8_t *buf_end;
>> +    const uint8_t *buf_ptr;
>> +    AVFrame *frm = (AVFrame*)data;
>> +    int left_bytes;
>> +    int ret, finish = 0;
>> +
>> +    *got_frame = 0;
>> +    frm->pts = -1;
>> +    frm->pict_type = AV_PICTURE_TYPE_NONE;
>> +
>> +    if (h->got_seqhdr) {
>> +        if (!frm->data[0] && (ret = ff_get_buffer(avctx, frm, 0)) < 0) {
>> +            return ret;
>> +        } 
>> +        h->dec_frame.priv = data;   // AVFrame
>> +    }
>> +
>> +    if (!buf_size) {
>> +        do {
>> +            ret = uavs3d_flush(h->dec_handle, &h->dec_frame);
>> +        } while (ret > 0 && !h->dec_frame.got_pic);
>> +    } else {
>> +        buf_ptr = buf;
>> +        buf_end = buf + buf_size;
>> +  
>> +        while (!finish) {
>> +            int bs_len;
>> +            uavs3d_io_frm_t *frm_dec = &h->dec_frame;
>> +
>> +            if (uavs3d_find_next_start_code(buf_ptr, buf_end - buf_ptr, &left_bytes)) {
>> +                bs_len = buf_end - buf_ptr - left_bytes;
>> +            } else { 
>> +                bs_len = buf_end - buf_ptr;
>> +                finish = 1;
>> +            }
>> +            frm_dec->bs = (unsigned char *)buf_ptr;
>> +            frm_dec->bs_len = bs_len;
>> +            frm_dec->pts = avpkt->pts;
>> +            frm_dec->dts = avpkt->dts;
>> +            uavs3d_decode(h->dec_handle, frm_dec);
>> +            buf_ptr += bs_len;
>> +
>> +            if (frm_dec->nal_type == NAL_SEQ_HEADER) {
>> +                static const int avs3_fps_num[9] = {0, 240000, 24, 25, 30000, 30, 50, 60000, 60 };
>> +                static const int avs3_fps_den[9] = {1,   1001,  1,  1,  1001,  1,  1,  1001,  1 };
>> +                avctx->framerate.num = avs3_fps_num[frm_dec->seqhdr->frame_rate_code];
>> +                avctx->framerate.den = avs3_fps_den[frm_dec->seqhdr->frame_rate_code];
>> +                avctx->has_b_frames = 1;
>> +                avctx->pix_fmt = frm_dec->seqhdr->bit_depth_internal == 8 ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_YUV420P10LE;
>> +                ff_set_dimensions(avctx, frm_dec->seqhdr->horizontal_size, frm_dec->seqhdr->vertical_size);
>> +                h->got_seqhdr = 1;
>> +            }
>> +            if (frm_dec->got_pic) {
>> +                break;
>> +            }
>> +        }
>> +    }
>> +
>> +    *got_frame = h->dec_frame.got_pic;
>> +
>> +    return buf_ptr - buf;
>> +}
>> +
>> +#define UAVS3D_OFFSET(x) offsetof(UAVS3DContext, x)
>> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
>> +static const AVOption options[] = {
>> +    { "frame_threads",      "number of frame-level threads ", UAVS3D_OFFSET(frame_threads),  AV_OPT_TYPE_INT,    {.i64 =  0 }, 0, UAVS3D_MAX_FRAME_THREADS, VE },
>
>No need for a custom option and UAVS3DContext field, you have
>avctx->threads for this.
>Just add the AV_CODEC_CAP_AUTO_THREADS capability to
>ff_libuavs3d_decoder and it will default to 0.

Will be fixed. Thanks.

>
>> +    { NULL }
>> +};
>> +static const AVClass libuavs3d_class = {
>> +    .class_name = "libuavs3d_class",
>> +    .item_name  = av_default_item_name,
>> +    .option     = options,
>> +    .version    = LIBAVUTIL_VERSION_INT,
>> +};
>> +
>> +AVCodec ff_libuavs3d_decoder = {
>> +    .name           = "libuavs3d",
>> +    .long_name      = NULL_IF_CONFIG_SMALL("uavs3d AVS3-P2/IEEE1857.10 decoder"),
>> +    .type           = AVMEDIA_TYPE_VIDEO,
>> +    .id             = AV_CODEC_ID_AVS3,
>> +    .priv_data_size = sizeof(UAVS3DContext),
>> +    .priv_class     = &libuavs3d_class,
>> +    .init           = libuavs3d_init,
>> +    .close          = libuavs3d_end,
>> +    .decode         = libuavs3d_decode_frame,
>> +    .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY,
>> +    .flush          = libuavs3d_flush,
>> +    .pix_fmts        = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_NONE },
>> +    
>> +    .wrapper_name   = "libuavs3d",
>> +};
>> 
>
>_______________________________________________
>ffmpeg-devel mailing list
>ffmpeg-devel@ffmpeg.org
>https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
>To unsubscribe, visit link above, or email
>ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Mark Thompson Aug. 11, 2020, 10:18 p.m. UTC | #3
On 05/08/2020 17:18, hwrenx@126.com wrote:
> From: hwren <hwrenx@126.com>
> 
> Signed-off-by: hbj <hanbj@pku.edu.cn>
> Signed-off-by: hwren <hwrenx@126.com>
> ---
>   Changelog              |   1 +
>   configure              |   4 +
>   doc/decoders.texi      |  21 +++
>   doc/general.texi       |   8 ++
>   libavcodec/Makefile    |   1 +
>   libavcodec/allcodecs.c |   1 +
>   libavcodec/libuavs3d.c | 283 +++++++++++++++++++++++++++++++++++++++++
>   7 files changed, 319 insertions(+)
>   create mode 100644 libavcodec/libuavs3d.c
> 
> ...
> diff --git a/libavcodec/libuavs3d.c b/libavcodec/libuavs3d.c
> new file mode 100644
> index 0000000000..c0d89cd1ce
> --- /dev/null
> +++ b/libavcodec/libuavs3d.c
> @@ -0,0 +1,283 @@
> +/*
> + * RAW AVS3-P2/IEEE1857.10 video demuxer
> + * Copyright (c) 2020 Zhenyu Wang <wangzhenyu@pkusz.edu.cn>
> + *                    Bingjie Han <hanbj@pkusz.edu.cn>
> + *                    Huiwen Ren  <hwrenx@gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/avutil.h"
> +#include "libavutil/common.h"
> +#include "libavutil/imgutils.h"
> +#include "libavutil/opt.h"
> +#include "avcodec.h"
> +#include "internal.h"
> +#include "uavs3d.h"
> +
> +#define UAVS3D_MAX_FRAME_THREADS 48

This restriction seems weird.  Is it reflecting some intrinsic property of the codec, or might it change in future?

> +
> +static const int color_primaries_tab[10] = {
> +    AVCOL_PRI_RESERVED0   ,    // 0
> +    AVCOL_PRI_BT709       ,    // 1
> +    AVCOL_PRI_UNSPECIFIED ,    // 2
> +    AVCOL_PRI_RESERVED    ,    // 3
> +    AVCOL_PRI_BT470M      ,    // 4
> +    AVCOL_PRI_BT470BG      ,    // 5
> +    AVCOL_PRI_SMPTE170M   ,    // 6
> +    AVCOL_PRI_SMPTE240M   ,    // 7
> +    AVCOL_PRI_FILM        ,    // 8
> +    AVCOL_PRI_BT2020           // 9
> +};
> +
> +static const int color_transfer_tab[15] = {
> +    AVCOL_TRC_RESERVED0    , // 0
> +    AVCOL_TRC_BT709        , // 1
> +    AVCOL_TRC_UNSPECIFIED  , // 2
> +    AVCOL_TRC_RESERVED     , // 3
> +    AVCOL_TRC_GAMMA22      , // 4
> +    AVCOL_TRC_GAMMA28      , // 5
> +    AVCOL_TRC_SMPTE170M    , // 6
> +    AVCOL_TRC_SMPTE240M    , // 7
> +    AVCOL_TRC_LINEAR       , // 8
> +    AVCOL_TRC_LOG          , // 9
> +    AVCOL_TRC_LOG_SQRT     , // 10
> +    AVCOL_TRC_BT2020_12    , // 11
> +    AVCOL_TRC_SMPTE2084    , // 12
> +    AVCOL_TRC_UNSPECIFIED  , // 13
> +    AVCOL_TRC_ARIB_STD_B67   // 14
> +};
> +
> +static const int color_matrix_tab[12] = {
> +    AVCOL_SPC_RESERVED     , // 0
> +    AVCOL_SPC_BT709        , // 1
> +    AVCOL_SPC_UNSPECIFIED  , // 2
> +    AVCOL_SPC_RESERVED     , // 3
> +    AVCOL_SPC_FCC          , // 4
> +    AVCOL_SPC_BT470BG      , // 5
> +    AVCOL_SPC_SMPTE170M    , // 6
> +    AVCOL_SPC_SMPTE240M    , // 7
> +    AVCOL_SPC_BT2020_NCL   , // 8
> +    AVCOL_SPC_BT2020_CL    , // 9
> +    AVCOL_SPC_UNSPECIFIED  , // 10
> +    AVCOL_SPC_UNSPECIFIED    // 11
> +};

These tables aren't used anywhere.

> +
> +static const enum AVPictureType IMGTYPE[8] = {

This name is doesn't conform to the usual naming convention.  uavs3_frame_types[], maybe?

> +    AV_PICTURE_TYPE_NONE,
> +    AV_PICTURE_TYPE_I,
> +    AV_PICTURE_TYPE_P,
> +    AV_PICTURE_TYPE_B
> +};

What do the other four values in the table mean?

> +
> +typedef struct UAVS3DContext {
> +    AVCodecContext  *avctx;
> +    void            *dec_handle;
> +    int              frame_threads;
> +    int              got_seqhdr;
> +    uavs3d_io_frm_t  dec_frame;
> +} UAVS3DContext;
> +
> +
> +static int uavs3d_find_next_start_code(const unsigned char *bs_data, int bs_len, int *left)
> +{
> +    const unsigned char *data_ptr = bs_data + 4;
> +    int count = bs_len - 4;
> +
> +    while (count >= 4 &&
> +        ((*(unsigned int *)data_ptr) != 0xB6010000) && /* P/B picture */

This pointer cast will be undefined behaviour, because the data need not be aligned.  Even if it were valid, it's still going to give the wrong answer on a machine with different endianness to the one you tested on.  Use AV_RB32() or AV_RL32().

> +        ((*(unsigned int *)data_ptr) != 0xB3010000) && /* I   picture */
> +        ((*(unsigned int *)data_ptr) != 0xB0010000) && /* sequence header */
> +        ((*(unsigned int *)data_ptr) != 0x00010000) && /* first slice */
> +        ((*(unsigned int *)data_ptr) != 0xB1010000)) { /* sequence end */

Make symbolic constants for the magic numbers here.

> +        data_ptr++;
> +        count--;
> +    }
> +
> +    if (count >= 4) {
> +        *left = count;
> +        return 1;
> +    }
> +
> +    return 0;
> +}
> +
> +static void uavs3d_output_callback(uavs3d_io_frm_t *dec_frame) {
> +    uavs3d_io_frm_t frm_out;
> +    AVFrame *frm = (AVFrame *)dec_frame->priv;
> +    int i;
> +
> +    if (!frm) {
> +        return;
> +    }
> +
> +    frm->pts       = dec_frame->pts;
> +    frm->pkt_dts   = dec_frame->dts;
> +    frm->pict_type = IMGTYPE[dec_frame->type];

Is overflow possible?

> +    frm->key_frame = (frm->pict_type == AV_PICTURE_TYPE_I);
> +
> +    for (i = 0; i < 3; i++) {
> +        frm_out.width [i] = dec_frame->width[i];
> +        frm_out.height[i] = dec_frame->height[i];
> +        frm_out.stride[i] = frm->linesize[i];
> +        frm_out.buffer[i] = frm->data[i];
> +    }
> +
> +    uavs3d_img_cpy_cvt(&frm_out, dec_frame, dec_frame->bit_depth);

Is this copy really required?  Decoders will normally allow you to reference the frame they hold internally.

Can the call ever fail?

> +}
> +
> +static av_cold int libuavs3d_init(AVCodecContext *avctx)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +    uavs3d_cfg_t cdsc;
> +
> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
> +    cdsc.check_md5 = 0;
> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);

Can this call fail?  (For example, if out of memory.)

> +    h->got_seqhdr = 0;
> +
> +    return 0;
> +}
> +
> +static av_cold int libuavs3d_end(AVCodecContext *avctx)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +
> +    if (h->dec_handle) {
> +        uavs3d_flush(h->dec_handle, NULL);
> +        uavs3d_delete(h->dec_handle);
> +        h->dec_handle = NULL;
> +    }
> +    h->got_seqhdr = 0;
> +
> +    return 0;
> +}
> +
> +static void libuavs3d_flush(AVCodecContext * avctx)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +    uavs3d_cfg_t cdsc;
> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
> +    cdsc.check_md5 = 0;
> +
> +    if (h->dec_handle) {
> +        uavs3d_flush(h->dec_handle, NULL);
> +        uavs3d_delete(h->dec_handle);
> +    }
> +
> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
> +    h->got_seqhdr = 0;
> +}
> +
> +static int libuavs3d_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
> +{
> +    UAVS3DContext *h = avctx->priv_data;
> +    const uint8_t *buf = avpkt->data;
> +    int buf_size = avpkt->size;
> +    const uint8_t *buf_end;
> +    const uint8_t *buf_ptr;
> +    AVFrame *frm = (AVFrame*)data;

No need for explicit casts from void*.

> +    int left_bytes;
> +    int ret, finish = 0;
> +
> +    *got_frame = 0;
> +    frm->pts = -1;
> +    frm->pict_type = AV_PICTURE_TYPE_NONE;
> +
> +    if (h->got_seqhdr) {
> +        if (!frm->data[0] && (ret = ff_get_buffer(avctx, frm, 0)) < 0) {
> +            return ret;
> +        }
> +        h->dec_frame.priv = data;   // AVFrame

Always allocating here if you have sequence header looks suspicious.  What if you don't get any frame output?

> +    }
> +
> +    if (!buf_size) {
> +        do {
> +            ret = uavs3d_flush(h->dec_handle, &h->dec_frame);
> +        } while (ret > 0 && !h->dec_frame.got_pic);
> +    } else {
> +        buf_ptr = buf;
> +        buf_end = buf + buf_size;
> +
> +        while (!finish) {
> +            int bs_len;
> +            uavs3d_io_frm_t *frm_dec = &h->dec_frame;
> +
> +            if (uavs3d_find_next_start_code(buf_ptr, buf_end - buf_ptr, &left_bytes)) {
> +                bs_len = buf_end - buf_ptr - left_bytes;
> +            } else {
> +                bs_len = buf_end - buf_ptr;
> +                finish = 1;
> +            }
> +            frm_dec->bs = (unsigned char *)buf_ptr;
> +            frm_dec->bs_len = bs_len;
> +            frm_dec->pts = avpkt->pts;
> +            frm_dec->dts = avpkt->dts;
> +            uavs3d_decode(h->dec_handle, frm_dec);

Does this call return any sort of error?

More generally, what happens if there is an error in decoding?  (For example, because the stream is corrupted.)

> +            buf_ptr += bs_len;
> +
> +            if (frm_dec->nal_type == NAL_SEQ_HEADER) {
> +                static const int avs3_fps_num[9] = {0, 240000, 24, 25, 30000, 30, 50, 60000, 60 };
> +                static const int avs3_fps_den[9] = {1,   1001,  1,  1,  1001,  1,  1,  1001,  1 };
> +                avctx->framerate.num = avs3_fps_num[frm_dec->seqhdr->frame_rate_code];
> +                avctx->framerate.den = avs3_fps_den[frm_dec->seqhdr->frame_rate_code];

Same comment as in 2/4 about ff_mpeg12_frame_rate_tab[].

> +                avctx->has_b_frames = 1;

Always?

> +                avctx->pix_fmt = frm_dec->seqhdr->bit_depth_internal == 8 ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_YUV420P10LE;
> +                ff_set_dimensions(avctx, frm_dec->seqhdr->horizontal_size, frm_dec->seqhdr->vertical_size);
> +                h->got_seqhdr = 1;
> +            }
> +            if (frm_dec->got_pic) {
> +                break;
> +            }
> +        }
> +    }
> +
> +    *got_frame = h->dec_frame.got_pic;
> +
> +    return buf_ptr - buf;
> +}
> +
> +#define UAVS3D_OFFSET(x) offsetof(UAVS3DContext, x)
> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
> +static const AVOption options[] = {
> +    { "frame_threads",      "number of frame-level threads ", UAVS3D_OFFSET(frame_threads),  AV_OPT_TYPE_INT,    {.i64 =  0 }, 0, UAVS3D_MAX_FRAME_THREADS, VE },
> +    { NULL }
> +};
> +static const AVClass libuavs3d_class = {
> +    .class_name = "libuavs3d_class",
> +    .item_name  = av_default_item_name,
> +    .option     = options,
> +    .version    = LIBAVUTIL_VERSION_INT,
> +};
> +
> +AVCodec ff_libuavs3d_decoder = {
> +    .name           = "libuavs3d",
> +    .long_name      = NULL_IF_CONFIG_SMALL("uavs3d AVS3-P2/IEEE1857.10 decoder"),
> +    .type           = AVMEDIA_TYPE_VIDEO,
> +    .id             = AV_CODEC_ID_AVS3,
> +    .priv_data_size = sizeof(UAVS3DContext),
> +    .priv_class     = &libuavs3d_class,
> +    .init           = libuavs3d_init,
> +    .close          = libuavs3d_end,
> +    .decode         = libuavs3d_decode_frame,
> +    .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY,
> +    .flush          = libuavs3d_flush,
> +    .pix_fmts        = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_NONE },
> +
> +    .wrapper_name   = "libuavs3d",
> +};
> 

- Mark
hwren Aug. 17, 2020, 11:42 a.m. UTC | #4
Fix the mail format of reply message from <hbj515 at sina.com> and reset it to the correct thread.
Same content.















At 2020-08-12 06:18:46, "Mark Thompson" <sw@jkqxz.net> wrote:
>On 05/08/2020 17:18, hwrenx@126.com wrote:
>> From: hwren <hwrenx@126.com>
>> 
>> Signed-off-by: hbj <hanbj@pku.edu.cn>
>> Signed-off-by: hwren <hwrenx@126.com>
>> ---
>>   Changelog              |   1 +
>>   configure              |   4 +
>>   doc/decoders.texi      |  21 +++
>>   doc/general.texi       |   8 ++
>>   libavcodec/Makefile    |   1 +
>>   libavcodec/allcodecs.c |   1 +
>>   libavcodec/libuavs3d.c | 283 +++++++++++++++++++++++++++++++++++++++++
>>   7 files changed, 319 insertions(+)
>>   create mode 100644 libavcodec/libuavs3d.c
>> 
>> ...
>> diff --git a/libavcodec/libuavs3d.c b/libavcodec/libuavs3d.c
>> new file mode 100644
>> index 0000000000..c0d89cd1ce
>> --- /dev/null
>> +++ b/libavcodec/libuavs3d.c
>> @@ -0,0 +1,283 @@
>> +/*
>> + * RAW AVS3-P2/IEEE1857.10 video demuxer
>> + * Copyright (c) 2020 Zhenyu Wang <wangzhenyu@pkusz.edu.cn>
>> + *                    Bingjie Han <hanbj@pkusz.edu.cn>
>> + *                    Huiwen Ren  <hwrenx@gmail.com>
>> + *
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
>> + */
>> +
>> +#include "libavutil/avassert.h"
>> +#include "libavutil/avutil.h"
>> +#include "libavutil/common.h"
>> +#include "libavutil/imgutils.h"
>> +#include "libavutil/opt.h"
>> +#include "avcodec.h"
>> +#include "internal.h"
>> +#include "uavs3d.h"
>> +
>> +#define UAVS3D_MAX_FRAME_THREADS 48
>
>This restriction seems weird.  Is it reflecting some intrinsic property of the codec, or might it change in future?

The restriction will be removed in the next version.

>
>> +
>> +static const int color_primaries_tab[10] = {
>> +    AVCOL_PRI_RESERVED0   ,    // 0
>> +    AVCOL_PRI_BT709       ,    // 1
>> +    AVCOL_PRI_UNSPECIFIED ,    // 2
>> +    AVCOL_PRI_RESERVED    ,    // 3
>> +    AVCOL_PRI_BT470M      ,    // 4
>> +    AVCOL_PRI_BT470BG      ,    // 5
>> +    AVCOL_PRI_SMPTE170M   ,    // 6
>> +    AVCOL_PRI_SMPTE240M   ,    // 7
>> +    AVCOL_PRI_FILM        ,    // 8
>> +    AVCOL_PRI_BT2020           // 9
>> +};
>> +
>> +static const int color_transfer_tab[15] = {
>> +    AVCOL_TRC_RESERVED0    , // 0
>> +    AVCOL_TRC_BT709        , // 1
>> +    AVCOL_TRC_UNSPECIFIED  , // 2
>> +    AVCOL_TRC_RESERVED     , // 3
>> +    AVCOL_TRC_GAMMA22      , // 4
>> +    AVCOL_TRC_GAMMA28      , // 5
>> +    AVCOL_TRC_SMPTE170M    , // 6
>> +    AVCOL_TRC_SMPTE240M    , // 7
>> +    AVCOL_TRC_LINEAR       , // 8
>> +    AVCOL_TRC_LOG          , // 9
>> +    AVCOL_TRC_LOG_SQRT     , // 10
>> +    AVCOL_TRC_BT2020_12    , // 11
>> +    AVCOL_TRC_SMPTE2084    , // 12
>> +    AVCOL_TRC_UNSPECIFIED  , // 13
>> +    AVCOL_TRC_ARIB_STD_B67   // 14
>> +};
>> +
>> +static const int color_matrix_tab[12] = {
>> +    AVCOL_SPC_RESERVED     , // 0
>> +    AVCOL_SPC_BT709        , // 1
>> +    AVCOL_SPC_UNSPECIFIED  , // 2
>> +    AVCOL_SPC_RESERVED     , // 3
>> +    AVCOL_SPC_FCC          , // 4
>> +    AVCOL_SPC_BT470BG      , // 5
>> +    AVCOL_SPC_SMPTE170M    , // 6
>> +    AVCOL_SPC_SMPTE240M    , // 7
>> +    AVCOL_SPC_BT2020_NCL   , // 8
>> +    AVCOL_SPC_BT2020_CL    , // 9
>> +    AVCOL_SPC_UNSPECIFIED  , // 10
>> +    AVCOL_SPC_UNSPECIFIED    // 11
>> +};
>
>These tables aren't used anywhere.

These informations are located in the sequence extension header, and the tables are ID maps between FFMPEG ID ans AVS3 ID.
We will add decoding these informations in the next version.

>
>> +
>> +static const enum AVPictureType IMGTYPE[8] = {
>
>This name is doesn't conform to the usual naming convention.  uavs3_frame_types[], maybe?
>
>> +    AV_PICTURE_TYPE_NONE,
>> +    AV_PICTURE_TYPE_I,
>> +    AV_PICTURE_TYPE_P,
>> +    AV_PICTURE_TYPE_B
>> +};
>
>What do the other four values in the table mean?

Sorry, it is a mistake. It will be fixed.

>
>> +
>> +typedef struct UAVS3DContext {
>> +    AVCodecContext  *avctx;
>> +    void            *dec_handle;
>> +    int              frame_threads;
>> +    int              got_seqhdr;
>> +    uavs3d_io_frm_t  dec_frame;
>> +} UAVS3DContext;
>> +
>> +
>> +static int uavs3d_find_next_start_code(const unsigned char *bs_data, int bs_len, int *left)
>> +{
>> +    const unsigned char *data_ptr = bs_data + 4;
>> +    int count = bs_len - 4;
>> +
>> +    while (count >= 4 &&
>> +        ((*(unsigned int *)data_ptr) != 0xB6010000) && /* P/B picture */
>
>This pointer cast will be undefined behaviour, because the data need not be aligned.  Even if it were valid, it's still going to give the wrong answer on a machine with different endianness to the one you tested on.  Use AV_RB32() or AV_RL32().

Thanks, we will use AV_RL32() instead of it.

>
>> +        ((*(unsigned int *)data_ptr) != 0xB3010000) && /* I   picture */
>> +        ((*(unsigned int *)data_ptr) != 0xB0010000) && /* sequence header */
>> +        ((*(unsigned int *)data_ptr) != 0x00010000) && /* first slice */
>> +        ((*(unsigned int *)data_ptr) != 0xB1010000)) { /* sequence end */
>
>Make symbolic constants for the magic numbers here.

Thanks, these will be fixed.

>
>> +        data_ptr++;
>> +        count--;
>> +    }
>> +
>> +    if (count >= 4) {
>> +        *left = count;
>> +        return 1;
>> +    }
>> +
>> +    return 0;
>> +}
>> +
>> +static void uavs3d_output_callback(uavs3d_io_frm_t *dec_frame) {
>> +    uavs3d_io_frm_t frm_out;
>> +    AVFrame *frm = (AVFrame *)dec_frame->priv;
>> +    int i;
>> +
>> +    if (!frm) {
>> +        return;
>> +    }
>> +
>> +    frm->pts       = dec_frame->pts;
>> +    frm->pkt_dts   = dec_frame->dts;
>> +    frm->pict_type = IMGTYPE[dec_frame->type];
>
>Is overflow possible?

It will be fixed.

>
>> +    frm->key_frame = (frm->pict_type == AV_PICTURE_TYPE_I);
>> +
>> +    for (i = 0; i < 3; i++) {
>> +        frm_out.width [i] = dec_frame->width[i];
>> +        frm_out.height[i] = dec_frame->height[i];
>> +        frm_out.stride[i] = frm->linesize[i];
>> +        frm_out.buffer[i] = frm->data[i];
>> +    }
>> +
>> +    uavs3d_img_cpy_cvt(&frm_out, dec_frame, dec_frame->bit_depth);
>
>Is this copy really required?  Decoders will normally allow you to reference the frame they hold internally.
>
>Can the call ever fail?

Sorry, the decoder will reuse the buffer of dec_frame in the next decoding, so the copy is necessory.
If the call "uavs3d_img_cpy_cvt" fail, an error image will be display. The error code is useless, and 
it not easy for error check with SIMD instructions for various CPUs.
We will check if the buffer is vaild in the next version. If not, set the got_pic flag with false.

>
>> +}
>> +
>> +static av_cold int libuavs3d_init(AVCodecContext *avctx)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +    uavs3d_cfg_t cdsc;
>> +
>> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
>> +    cdsc.check_md5 = 0;
>> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
>
>Can this call fail?  (For example, if out of memory.)

Thanks, it will be fixed.

>
>> +    h->got_seqhdr = 0;
>> +
>> +    return 0;
>> +}
>> +
>> +static av_cold int libuavs3d_end(AVCodecContext *avctx)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +
>> +    if (h->dec_handle) {
>> +        uavs3d_flush(h->dec_handle, NULL);
>> +        uavs3d_delete(h->dec_handle);
>> +        h->dec_handle = NULL;
>> +    }
>> +    h->got_seqhdr = 0;
>> +
>> +    return 0;
>> +}
>> +
>> +static void libuavs3d_flush(AVCodecContext * avctx)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +    uavs3d_cfg_t cdsc;
>> +    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
>> +    cdsc.check_md5 = 0;
>> +
>> +    if (h->dec_handle) {
>> +        uavs3d_flush(h->dec_handle, NULL);
>> +        uavs3d_delete(h->dec_handle);
>> +    }
>> +
>> +    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
>> +    h->got_seqhdr = 0;
>> +}
>> +
>> +static int libuavs3d_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
>> +{
>> +    UAVS3DContext *h = avctx->priv_data;
>> +    const uint8_t *buf = avpkt->data;
>> +    int buf_size = avpkt->size;
>> +    const uint8_t *buf_end;
>> +    const uint8_t *buf_ptr;
>> +    AVFrame *frm = (AVFrame*)data;
>
>No need for explicit casts from void*.

Thanks, it will be fixed.

>
>> +    int left_bytes;
>> +    int ret, finish = 0;
>> +
>> +    *got_frame = 0;
>> +    frm->pts = -1;
>> +    frm->pict_type = AV_PICTURE_TYPE_NONE;
>> +
>> +    if (h->got_seqhdr) {
>> +        if (!frm->data[0] && (ret = ff_get_buffer(avctx, frm, 0)) < 0) {
>> +            return ret;
>> +        }
>> +        h->dec_frame.priv = data;   // AVFrame
>
>Always allocating here if you have sequence header looks suspicious.  What if you don't get any frame output?

Thanks, we will free the buffer in the next version if no frame is outputed. We don't know whether one frame will be got before
decoding, so it is a simple but effective implementation to allocate the buffer once the frame size and pix_fmt are received. 

>
>> +    }
>> +
>> +    if (!buf_size) {
>> +        do {
>> +            ret = uavs3d_flush(h->dec_handle, &h->dec_frame);
>> +        } while (ret > 0 && !h->dec_frame.got_pic);
>> +    } else {
>> +        buf_ptr = buf;
>> +        buf_end = buf + buf_size;
>> +
>> +        while (!finish) {
>> +            int bs_len;
>> +            uavs3d_io_frm_t *frm_dec = &h->dec_frame;
>> +
>> +            if (uavs3d_find_next_start_code(buf_ptr, buf_end - buf_ptr, &left_bytes)) {
>> +                bs_len = buf_end - buf_ptr - left_bytes;
>> +            } else {
>> +                bs_len = buf_end - buf_ptr;
>> +                finish = 1;
>> +            }
>> +            frm_dec->bs = (unsigned char *)buf_ptr;
>> +            frm_dec->bs_len = bs_len;
>> +            frm_dec->pts = avpkt->pts;
>> +            frm_dec->dts = avpkt->dts;
>> +            uavs3d_decode(h->dec_handle, frm_dec);
>
>Does this call return any sort of error?
>
>More generally, what happens if there is an error in decoding?  (For example, because the stream is corrupted.)

Thanks, errors will be checked.

>
>> +            buf_ptr += bs_len;
>> +
>> +            if (frm_dec->nal_type == NAL_SEQ_HEADER) {
>> +                static const int avs3_fps_num[9] = {0, 240000, 24, 25, 30000, 30, 50, 60000, 60 };
>> +                static const int avs3_fps_den[9] = {1,   1001,  1,  1,  1001,  1,  1,  1001,  1 };
>> +                avctx->framerate.num = avs3_fps_num[frm_dec->seqhdr->frame_rate_code];
>> +                avctx->framerate.den = avs3_fps_den[frm_dec->seqhdr->frame_rate_code];
>
>Same comment as in 2/4 about ff_mpeg12_frame_rate_tab[].

It will be fixed.

>
>> +                avctx->has_b_frames = 1;
>
>Always?

In AVS3, no flag shows whether B frames exist, we will choose the low delay flag for it. 

>
>> +                avctx->pix_fmt = frm_dec->seqhdr->bit_depth_internal == 8 ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_YUV420P10LE;
>> +                ff_set_dimensions(avctx, frm_dec->seqhdr->horizontal_size, frm_dec->seqhdr->vertical_size);
>> +                h->got_seqhdr = 1;
>> +            }
>> +            if (frm_dec->got_pic) {
>> +                break;
>> +            }
>> +        }
>> +    }
>> +
>> +    *got_frame = h->dec_frame.got_pic;
>> +
>> +    return buf_ptr - buf;
>> +}
>> +
>> +#define UAVS3D_OFFSET(x) offsetof(UAVS3DContext, x)
>> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
>> +static const AVOption options[] = {
>> +    { "frame_threads",      "number of frame-level threads ", UAVS3D_OFFSET(frame_threads),  AV_OPT_TYPE_INT,    {.i64 =  0 }, 0, UAVS3D_MAX_FRAME_THREADS, VE },
>> +    { NULL }
>> +};
>> +static const AVClass libuavs3d_class = {
>> +    .class_name = "libuavs3d_class",
>> +    .item_name  = av_default_item_name,
>> +    .option     = options,
>> +    .version    = LIBAVUTIL_VERSION_INT,
>> +};
>> +
>> +AVCodec ff_libuavs3d_decoder = {
>> +    .name           = "libuavs3d",
>> +    .long_name      = NULL_IF_CONFIG_SMALL("uavs3d AVS3-P2/IEEE1857.10 decoder"),
>> +    .type           = AVMEDIA_TYPE_VIDEO,
>> +    .id             = AV_CODEC_ID_AVS3,
>> +    .priv_data_size = sizeof(UAVS3DContext),
>> +    .priv_class     = &libuavs3d_class,
>> +    .init           = libuavs3d_init,
>> +    .close          = libuavs3d_end,
>> +    .decode         = libuavs3d_decode_frame,
>> +    .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY,
>> +    .flush          = libuavs3d_flush,
>> +    .pix_fmts        = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_NONE },
>> +
>> +    .wrapper_name   = "libuavs3d",
>> +};
>> 
>
>- Mark
>_______________________________________________
>ffmpeg-devel mailing list
>ffmpeg-devel@ffmpeg.org
>https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
>To unsubscribe, visit link above, or email
>ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox series

Patch

diff --git a/Changelog b/Changelog
index a60e7d2eb8..dfd56b3fc6 100644
--- a/Changelog
+++ b/Changelog
@@ -4,6 +4,7 @@  releases are sorted from youngest to oldest.
 version <next>:
 - AudioToolbox output device
 - MacCaption demuxer
+- AVS3 video decoder via libuavs3d
 
 
 version 4.3:
diff --git a/configure b/configure
index 7495f35faa..7340bc4a35 100755
--- a/configure
+++ b/configure
@@ -274,6 +274,7 @@  External library support:
   --enable-libtls          enable LibreSSL (via libtls), needed for https support
                            if openssl, gnutls or mbedtls is not used [no]
   --enable-libtwolame      enable MP2 encoding via libtwolame [no]
+  --enable-libuavs3d       enable AVS3 decoding via libuavs3d [no]
   --enable-libv4l2         enable libv4l2/v4l-utils [no]
   --enable-libvidstab      enable video stabilization using vid.stab [no]
   --enable-libvmaf         enable vmaf filter via libvmaf [no]
@@ -1807,6 +1808,7 @@  EXTERNAL_LIBRARY_LIST="
     libtesseract
     libtheora
     libtwolame
+    libuavs3d
     libv4l2
     libvorbis
     libvpx
@@ -3242,6 +3244,7 @@  libspeex_encoder_deps="libspeex"
 libspeex_encoder_select="audio_frame_queue"
 libtheora_encoder_deps="libtheora"
 libtwolame_encoder_deps="libtwolame"
+libuavs3d_decoder_deps="libuavs3d"
 libvo_amrwbenc_encoder_deps="libvo_amrwbenc"
 libvorbis_decoder_deps="libvorbis"
 libvorbis_encoder_deps="libvorbis libvorbisenc"
@@ -6379,6 +6382,7 @@  enabled libtls            && require_pkg_config libtls libtls tls.h tls_configur
 enabled libtwolame        && require libtwolame twolame.h twolame_init -ltwolame &&
                              { check_lib libtwolame twolame.h twolame_encode_buffer_float32_interleaved -ltwolame ||
                                die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; }
+enabled libuavs3d         && require_pkg_config libuavs3d uavs3d uavs3d.h uavs3d_decode
 enabled libv4l2           && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_ioctl
 enabled libvidstab        && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit
 enabled libvmaf           && require_pkg_config libvmaf "libvmaf >= 1.3.9" libvmaf.h compute_vmaf
diff --git a/doc/decoders.texi b/doc/decoders.texi
index 9005714e3c..f1a0b3c36e 100644
--- a/doc/decoders.texi
+++ b/doc/decoders.texi
@@ -86,6 +86,27 @@  AVS2-P2/IEEE1857.4 video decoder wrapper.
 
 This decoder allows libavcodec to decode AVS2 streams with davs2 library.
 
+@c man end VIDEO DECODERS
+ 
+@section libuavs3d
+
+AVS3-P2/IEEE1857.10 video decoder.
+
+libuavs3d allows libavcodec to decode AVS3 streams.
+Requires the presence of the libuavs3d headers and library during configuration.
+You need to explicitly configure the build with @code{--enable-libuavs3d}.
+
+@subsection Options
+
+The following option is supported by the libuavs3d wrapper.
+
+@table @option
+
+@item frame_threads
+Set amount of frame threads to use during decoding. The default value is 0 (autodetect).
+
+@end table
+
 @c man end VIDEO DECODERS
 
 @chapter Audio Decoders
diff --git a/doc/general.texi b/doc/general.texi
index 9b0ee96752..6d673b74e1 100644
--- a/doc/general.texi
+++ b/doc/general.texi
@@ -125,6 +125,14 @@  Go to @url{https://github.com/pkuvcl/davs2} and follow the instructions for
 installing the library. Then pass @code{--enable-libdavs2} to configure to
 enable it.
 
+@section uavs3d
+
+FFmpeg can make use of the uavs3d library for AVS3-P2/IEEE1857.10 video decoding.
+
+Go to @url{https://github.com/uavs3/uavs3d} and follow the instructions for
+installing the library. Then pass @code{--enable-libuavs3d} to configure to
+enable it.
+
 @float NOTE
 libdavs2 is under the GNU Public License Version 2 or later
 (see @url{http://www.gnu.org/licenses/old-licenses/gpl-2.0.html} for
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index f1512779be..491485f3c0 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1026,6 +1026,7 @@  OBJS-$(CONFIG_LIBSPEEX_DECODER)           += libspeexdec.o
 OBJS-$(CONFIG_LIBSPEEX_ENCODER)           += libspeexenc.o
 OBJS-$(CONFIG_LIBTHEORA_ENCODER)          += libtheoraenc.o
 OBJS-$(CONFIG_LIBTWOLAME_ENCODER)         += libtwolame.o
+OBJS-$(CONFIG_LIBUAVS3D_DECODER)          += libuavs3d.o
 OBJS-$(CONFIG_LIBVO_AMRWBENC_ENCODER)     += libvo-amrwbenc.o
 OBJS-$(CONFIG_LIBVORBIS_DECODER)          += libvorbisdec.o
 OBJS-$(CONFIG_LIBVORBIS_ENCODER)          += libvorbisenc.o \
diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
index 80f128cade..3d2d0af87a 100644
--- a/libavcodec/allcodecs.c
+++ b/libavcodec/allcodecs.c
@@ -730,6 +730,7 @@  extern AVCodec ff_libspeex_encoder;
 extern AVCodec ff_libspeex_decoder;
 extern AVCodec ff_libtheora_encoder;
 extern AVCodec ff_libtwolame_encoder;
+extern AVCodec ff_libuavs3d_decoder;
 extern AVCodec ff_libvo_amrwbenc_encoder;
 extern AVCodec ff_libvorbis_encoder;
 extern AVCodec ff_libvorbis_decoder;
diff --git a/libavcodec/libuavs3d.c b/libavcodec/libuavs3d.c
new file mode 100644
index 0000000000..c0d89cd1ce
--- /dev/null
+++ b/libavcodec/libuavs3d.c
@@ -0,0 +1,283 @@ 
+/*
+ * RAW AVS3-P2/IEEE1857.10 video demuxer
+ * Copyright (c) 2020 Zhenyu Wang <wangzhenyu@pkusz.edu.cn>
+ *                    Bingjie Han <hanbj@pkusz.edu.cn>
+ *                    Huiwen Ren  <hwrenx@gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/avassert.h"
+#include "libavutil/avutil.h"
+#include "libavutil/common.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+#include "avcodec.h"
+#include "internal.h"
+#include "uavs3d.h"
+
+#define UAVS3D_MAX_FRAME_THREADS 48 
+
+static const int color_primaries_tab[10] = {
+    AVCOL_PRI_RESERVED0   ,    // 0
+    AVCOL_PRI_BT709       ,    // 1
+    AVCOL_PRI_UNSPECIFIED ,    // 2
+    AVCOL_PRI_RESERVED    ,    // 3
+    AVCOL_PRI_BT470M      ,    // 4
+    AVCOL_PRI_BT470BG      ,    // 5
+    AVCOL_PRI_SMPTE170M   ,    // 6
+    AVCOL_PRI_SMPTE240M   ,    // 7
+    AVCOL_PRI_FILM        ,    // 8
+    AVCOL_PRI_BT2020           // 9  
+};
+    
+static const int color_transfer_tab[15] = {
+    AVCOL_TRC_RESERVED0    , // 0
+    AVCOL_TRC_BT709        , // 1
+    AVCOL_TRC_UNSPECIFIED  , // 2
+    AVCOL_TRC_RESERVED     , // 3
+    AVCOL_TRC_GAMMA22      , // 4
+    AVCOL_TRC_GAMMA28      , // 5
+    AVCOL_TRC_SMPTE170M    , // 6
+    AVCOL_TRC_SMPTE240M    , // 7
+    AVCOL_TRC_LINEAR       , // 8
+    AVCOL_TRC_LOG          , // 9
+    AVCOL_TRC_LOG_SQRT     , // 10
+    AVCOL_TRC_BT2020_12    , // 11
+    AVCOL_TRC_SMPTE2084    , // 12
+    AVCOL_TRC_UNSPECIFIED  , // 13
+    AVCOL_TRC_ARIB_STD_B67   // 14
+};
+
+static const int color_matrix_tab[12] = {
+    AVCOL_SPC_RESERVED     , // 0
+    AVCOL_SPC_BT709        , // 1
+    AVCOL_SPC_UNSPECIFIED  , // 2
+    AVCOL_SPC_RESERVED     , // 3
+    AVCOL_SPC_FCC          , // 4
+    AVCOL_SPC_BT470BG      , // 5
+    AVCOL_SPC_SMPTE170M    , // 6
+    AVCOL_SPC_SMPTE240M    , // 7
+    AVCOL_SPC_BT2020_NCL   , // 8
+    AVCOL_SPC_BT2020_CL    , // 9
+    AVCOL_SPC_UNSPECIFIED  , // 10
+    AVCOL_SPC_UNSPECIFIED    // 11
+};
+
+static const enum AVPictureType IMGTYPE[8] = {
+    AV_PICTURE_TYPE_NONE,
+    AV_PICTURE_TYPE_I,
+    AV_PICTURE_TYPE_P,
+    AV_PICTURE_TYPE_B
+};
+
+typedef struct UAVS3DContext {
+    AVCodecContext  *avctx;
+    void            *dec_handle;
+    int              frame_threads;
+    int              got_seqhdr;
+    uavs3d_io_frm_t  dec_frame;
+} UAVS3DContext;
+
+
+static int uavs3d_find_next_start_code(const unsigned char *bs_data, int bs_len, int *left)
+{
+    const unsigned char *data_ptr = bs_data + 4;
+    int count = bs_len - 4;
+
+    while (count >= 4 &&
+        ((*(unsigned int *)data_ptr) != 0xB6010000) && /* P/B picture */
+        ((*(unsigned int *)data_ptr) != 0xB3010000) && /* I   picture */
+        ((*(unsigned int *)data_ptr) != 0xB0010000) && /* sequence header */
+        ((*(unsigned int *)data_ptr) != 0x00010000) && /* first slice */
+        ((*(unsigned int *)data_ptr) != 0xB1010000)) { /* sequence end */
+        data_ptr++;
+        count--;
+    }
+
+    if (count >= 4) {
+        *left = count; 
+        return 1;
+    }
+
+    return 0;
+}
+
+static void uavs3d_output_callback(uavs3d_io_frm_t *dec_frame) {
+    uavs3d_io_frm_t frm_out;
+    AVFrame *frm = (AVFrame *)dec_frame->priv;
+    int i;
+
+    if (!frm) {
+        return;
+    }
+    
+    frm->pts       = dec_frame->pts;
+    frm->pkt_dts   = dec_frame->dts;
+    frm->pict_type = IMGTYPE[dec_frame->type];
+    frm->key_frame = (frm->pict_type == AV_PICTURE_TYPE_I);
+
+    for (i = 0; i < 3; i++) {
+        frm_out.width [i] = dec_frame->width[i];
+        frm_out.height[i] = dec_frame->height[i];
+        frm_out.stride[i] = frm->linesize[i];
+        frm_out.buffer[i] = frm->data[i]; 
+    }
+
+    uavs3d_img_cpy_cvt(&frm_out, dec_frame, dec_frame->bit_depth);
+}
+
+static av_cold int libuavs3d_init(AVCodecContext *avctx)
+{
+    UAVS3DContext *h = avctx->priv_data;
+    uavs3d_cfg_t cdsc;
+
+    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
+    cdsc.check_md5 = 0;
+    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
+    h->got_seqhdr = 0;
+ 
+    return 0;
+}
+
+static av_cold int libuavs3d_end(AVCodecContext *avctx)
+{
+    UAVS3DContext *h = avctx->priv_data;
+    
+    if (h->dec_handle) {
+        uavs3d_flush(h->dec_handle, NULL);
+        uavs3d_delete(h->dec_handle);
+        h->dec_handle = NULL;
+    }
+    h->got_seqhdr = 0;
+    
+    return 0;
+}
+
+static void libuavs3d_flush(AVCodecContext * avctx)
+{
+    UAVS3DContext *h = avctx->priv_data;
+    uavs3d_cfg_t cdsc;
+    cdsc.frm_threads = FFMIN(h->frame_threads > 0 ? h->frame_threads : av_cpu_count(), UAVS3D_MAX_FRAME_THREADS);
+    cdsc.check_md5 = 0;
+
+    if (h->dec_handle) {
+        uavs3d_flush(h->dec_handle, NULL);
+        uavs3d_delete(h->dec_handle);
+    }
+     
+    h->dec_handle = uavs3d_create(&cdsc, uavs3d_output_callback, NULL);
+    h->got_seqhdr = 0;
+}
+
+static int libuavs3d_decode_frame(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
+{
+    UAVS3DContext *h = avctx->priv_data;
+    const uint8_t *buf = avpkt->data;
+    int buf_size = avpkt->size;
+    const uint8_t *buf_end;
+    const uint8_t *buf_ptr;
+    AVFrame *frm = (AVFrame*)data;
+    int left_bytes;
+    int ret, finish = 0;
+
+    *got_frame = 0;
+    frm->pts = -1;
+    frm->pict_type = AV_PICTURE_TYPE_NONE;
+
+    if (h->got_seqhdr) {
+        if (!frm->data[0] && (ret = ff_get_buffer(avctx, frm, 0)) < 0) {
+            return ret;
+        } 
+        h->dec_frame.priv = data;   // AVFrame
+    }
+
+    if (!buf_size) {
+        do {
+            ret = uavs3d_flush(h->dec_handle, &h->dec_frame);
+        } while (ret > 0 && !h->dec_frame.got_pic);
+    } else {
+        buf_ptr = buf;
+        buf_end = buf + buf_size;
+  
+        while (!finish) {
+            int bs_len;
+            uavs3d_io_frm_t *frm_dec = &h->dec_frame;
+
+            if (uavs3d_find_next_start_code(buf_ptr, buf_end - buf_ptr, &left_bytes)) {
+                bs_len = buf_end - buf_ptr - left_bytes;
+            } else { 
+                bs_len = buf_end - buf_ptr;
+                finish = 1;
+            }
+            frm_dec->bs = (unsigned char *)buf_ptr;
+            frm_dec->bs_len = bs_len;
+            frm_dec->pts = avpkt->pts;
+            frm_dec->dts = avpkt->dts;
+            uavs3d_decode(h->dec_handle, frm_dec);
+            buf_ptr += bs_len;
+
+            if (frm_dec->nal_type == NAL_SEQ_HEADER) {
+                static const int avs3_fps_num[9] = {0, 240000, 24, 25, 30000, 30, 50, 60000, 60 };
+                static const int avs3_fps_den[9] = {1,   1001,  1,  1,  1001,  1,  1,  1001,  1 };
+                avctx->framerate.num = avs3_fps_num[frm_dec->seqhdr->frame_rate_code];
+                avctx->framerate.den = avs3_fps_den[frm_dec->seqhdr->frame_rate_code];
+                avctx->has_b_frames = 1;
+                avctx->pix_fmt = frm_dec->seqhdr->bit_depth_internal == 8 ? AV_PIX_FMT_YUV420P : AV_PIX_FMT_YUV420P10LE;
+                ff_set_dimensions(avctx, frm_dec->seqhdr->horizontal_size, frm_dec->seqhdr->vertical_size);
+                h->got_seqhdr = 1;
+            }
+            if (frm_dec->got_pic) {
+                break;
+            }
+        }
+    }
+
+    *got_frame = h->dec_frame.got_pic;
+
+    return buf_ptr - buf;
+}
+
+#define UAVS3D_OFFSET(x) offsetof(UAVS3DContext, x)
+#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
+static const AVOption options[] = {
+    { "frame_threads",      "number of frame-level threads ", UAVS3D_OFFSET(frame_threads),  AV_OPT_TYPE_INT,    {.i64 =  0 }, 0, UAVS3D_MAX_FRAME_THREADS, VE },
+    { NULL }
+};
+static const AVClass libuavs3d_class = {
+    .class_name = "libuavs3d_class",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+AVCodec ff_libuavs3d_decoder = {
+    .name           = "libuavs3d",
+    .long_name      = NULL_IF_CONFIG_SMALL("uavs3d AVS3-P2/IEEE1857.10 decoder"),
+    .type           = AVMEDIA_TYPE_VIDEO,
+    .id             = AV_CODEC_ID_AVS3,
+    .priv_data_size = sizeof(UAVS3DContext),
+    .priv_class     = &libuavs3d_class,
+    .init           = libuavs3d_init,
+    .close          = libuavs3d_end,
+    .decode         = libuavs3d_decode_frame,
+    .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY,
+    .flush          = libuavs3d_flush,
+    .pix_fmts        = (const enum AVPixelFormat[]) { AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_NONE },
+    
+    .wrapper_name   = "libuavs3d",
+};