[FFmpeg-devel,GSoC,1/6] avformat/abr: Adaptive Bitrate support

From: spartazhc <spartazhc@gmail.com>

Add abr module for hls/dash.

Signed-off-by: spartazhc <spartazhc@gmail.com>
---
 doc/protocols.texi      |   7 +
 libavformat/Makefile    |   1 +
 libavformat/abr.c       | 282 ++++++++++++++++++++++++++++++++++++++++
 libavformat/protocols.c |   1 +
 4 files changed, 291 insertions(+)
 create mode 100644 libavformat/abr.c

Hongcheng Zhong <sj.hc_Zhong@sjtu.edu.cn> 于2020年7月5日周日 下午7:35写道：
>
> From: spartazhc <spartazhc@gmail.com>
>
> Add abr module for hls/dash.
>
> Signed-off-by: spartazhc <spartazhc@gmail.com>
> ---
>  doc/protocols.texi      |   7 +
>  libavformat/Makefile    |   1 +
>  libavformat/abr.c       | 282 ++++++++++++++++++++++++++++++++++++++++
>  libavformat/protocols.c |   1 +
>  4 files changed, 291 insertions(+)
>  create mode 100644 libavformat/abr.c
>
> diff --git a/doc/protocols.texi b/doc/protocols.texi
> index 644a17963d..fc80209884 100644
> --- a/doc/protocols.texi
> +++ b/doc/protocols.texi
> @@ -51,6 +51,13 @@ in microseconds.
>
>  A description of the currently available protocols follows.
>
> +@section abr
> +
> +Adaptive bitrate sub-protocol work for hls/dash.
> +
> +The abr protocol takes stream information from hls/dash as input,
> +use bandwidth estimation to decide whether to switch or not.
> +
>  @section amqp
>
>  Advanced Message Queueing Protocol (AMQP) version 0-9-1 is a broker based
> diff --git a/libavformat/Makefile b/libavformat/Makefile
> index 26af859a28..3c08289d5e 100644
> --- a/libavformat/Makefile
> +++ b/libavformat/Makefile
> @@ -588,6 +588,7 @@ OBJS-$(CONFIG_LIBOPENMPT_DEMUXER)        += libopenmpt.o
>  OBJS-$(CONFIG_VAPOURSYNTH_DEMUXER)       += vapoursynth.o
>
>  # protocols I/O
> +OBJS-$(CONFIG_ABR_PROTOCOL)              += abr.o
>  OBJS-$(CONFIG_ASYNC_PROTOCOL)            += async.o
>  OBJS-$(CONFIG_APPLEHTTP_PROTOCOL)        += hlsproto.o
>  OBJS-$(CONFIG_BLURAY_PROTOCOL)           += bluray.o
> diff --git a/libavformat/abr.c b/libavformat/abr.c
> new file mode 100644
> index 0000000000..b7d00efcae
> --- /dev/null
> +++ b/libavformat/abr.c
> @@ -0,0 +1,282 @@
> +/*
> + * Adaptive Bitrate Module for HLS / DASH
> + * Copyright (c) 2020 Hongcheng Zhong
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "avformat.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/time.h"
> +#include "libavutil/avstring.h"
> +#include "url.h"
> +#include <math.h>
> +
> +enum ABRFormatType {
> +    ABR_TYPE_HLS,
> +    ABR_TYPE_DASH
> +};
> +
> +typedef struct variant_bitrate {
> +    int value;
> +    int index;
> +} variant_bitrate;
> +
> +typedef struct ABRContext {
> +    AVClass *class;
> +    URLContext *hd;
> +    AVDictionary *abr_params;
> +    AVDictionary *abr_metadata;
> +    enum ABRFormatType format;
> +    int cur_pls;
> +    int can_switch;
> +    int n_variants;
> +    variant_bitrate *variants_bitrate;
> +    int index;
> +    int n_throughputs;
> +    float *throughputs;
> +    int64_t position;
> +} ABRContext;
> +
> +static float harmonic_mean(int num, float* arr)
> +{
> +    float tmp = 0;
> +
> +    if (num <= 0) return 0;
> +
> +    for (size_t i = 0; i < num; i++) {
> +        tmp += 1 / arr[i];
> +    }
> +
> +    return num / tmp;
> +}
> +
> +static int hls_param_parse(ABRContext *c, const char *key, const char *value)
> +{
> +    if (!av_strcasecmp(key, "cur_pls")) {
> +        c->cur_pls = atoi(value);
> +    } else if (!av_strcasecmp(key, "can_switch")) {
> +        c->can_switch = atoi(value);
> +    } else if (!av_strcasecmp(key, "n_variants")) {
> +        c->n_variants = atoi(value);
> +        c->variants_bitrate = av_mallocz(sizeof(variant_bitrate) * c->n_variants);
> +        if (!c->variants_bitrate)
> +            return -1;
> +    } else if (av_strstart(key, "variant_bitrate", NULL)) {
> +        c->variants_bitrate[c->index].value = atoi(value);
> +        c->variants_bitrate[c->index].index = c->index;
> +        c->index++;
> +    } else if (!av_strcasecmp(key, "n_throughputs")) {
> +        c->n_throughputs = atoi(value);
> +        c->index = 0;
> +        if (c->n_throughputs > 0) {
> +            c->throughputs = av_malloc(sizeof(float) * c->n_throughputs);
> +            if (!c->throughputs)
> +                return -1;
> +        }
> +    } else if (av_strstart(key, "throughputs", NULL))
> +        c->throughputs[c->index++] = atof(value);
> +    return 0;
> +}
> +
> +static int dash_param_parse(ABRContext *c, const char *key, const char *value)
> +{
> +    return 0;
> +}
> +
> +static int abr_param_parse(ABRContext *c, enum ABRFormatType type, const char *key, const char *value)
> +{
> +    if (type == ABR_TYPE_HLS) {
> +        hls_param_parse(c, key, value);
> +    } else if (type == ABR_TYPE_DASH) {
> +        dash_param_parse(c, key, value);
> +    }
> +    return 0;
> +}
> +
> +static int compare_vb(const void *a, const void *b)
> +{
> +    variant_bitrate *a1 = (variant_bitrate *)a;
> +    variant_bitrate *a2 = (variant_bitrate *)b;
> +    return (*a2).value - (*a1).value;
> +}
> +
> +static int abr_rule(ABRContext *c, float bw_estimate)
> +{
> +    int ret = -1;
> +
> +    if (c->n_throughputs > 6) {
> +        if (bw_estimate < c->variants_bitrate[c->cur_pls].value / 1000 * 1.2 &&
> +            bw_estimate > c->variants_bitrate[c->cur_pls].value / 1000 * 0.8)
> +            return -1;
> +        qsort(c->variants_bitrate, c->n_variants, sizeof(variant_bitrate), compare_vb);
> +        for (int i = 0; i < c->n_variants; i++) {
> +            if (bw_estimate > c->variants_bitrate[i].value / 1000) {
> +                ret = i;
> +                break;
> +            }
> +        }
> +    }
> +    if (ret == c->cur_pls)
> +        ret = -1;
> +    av_log(c, AV_LOG_VERBOSE, "[switch] bwe=%.2fkbps, cur=%d, switch=%d\n", bw_estimate, c->cur_pls, ret);
> +    return ret;
> +}
> +
> +static int abr_open(URLContext *h, const char *uri, int flags, AVDictionary **options)
> +{
> +    const char *nested_url;
> +    uint64_t start, end;
> +    float bw_estimation;
> +    int switch_request = -1;
> +    int ret = 0;
> +    ABRContext *c = h->priv_data;
> +
> +    if (!av_strstart(uri, "abr+", &nested_url) &&
> +        !av_strstart(uri, "abr:", &nested_url)) {
> +        av_log(h, AV_LOG_ERROR, "Unsupported url %s\n", uri);
> +        ret = AVERROR(EINVAL);
> +        goto err;
> +    }
> +
> +    {
this braces can be removed
> +        AVDictionaryEntry *en = NULL;
> +        en = av_dict_get(c->abr_params, "", en, AV_DICT_IGNORE_SUFFIX);
> +        if (!en)
> +            return -1;
> +        if (!av_strcasecmp(en->key, "format")) {
> +            if (!av_strcasecmp(en->value, "hls")) {
> +                c->format = ABR_TYPE_HLS;
> +            } else if (!av_strcasecmp(en->value, "dash")) {
> +                c->format = ABR_TYPE_DASH;
> +            }
> +            av_log(h, AV_LOG_VERBOSE, "%s is using ABR\n", en->value);
> +        }
> +
> +        while (en = av_dict_get(c->abr_params, "", en, AV_DICT_IGNORE_SUFFIX)) {
> +           if (abr_param_parse(c, c->format, en->key, en->value) < 0)
> +               av_log(h, AV_LOG_WARNING,
> +                      "Error parsing option '%s = %s'.\n",
> +                       en->key, en->value);
> +        }
> +    }
braces
> +
> +    start = av_gettime();
> +    if ((ret = ffurl_open_whitelist(&c->hd, nested_url, flags,
> +                                    &h->interrupt_callback, options,
> +                                    h->protocol_whitelist, h->protocol_blacklist, h)) < 0) {
> +        av_log(h, AV_LOG_ERROR, "Unable to open resource: %s\n", nested_url);
> +        goto err;
> +    }
> +    end = av_gettime();
> +
> +    bw_estimation = harmonic_mean(c->n_throughputs, c->throughputs);
> +
> +    if (c->can_switch == 1)
> +        switch_request = abr_rule(c, bw_estimation);
> +
> +    av_dict_set_int(&c->abr_metadata, "download_time", (end - start), 0);
> +    av_dict_set_int(&c->abr_metadata, "switch_request", switch_request, 0);
> +
> +err:
> +    return ret;
> +}
> +
> +
> +static int abr_read(URLContext *h, uint8_t *buf, int size)
> +{
> +    ABRContext *c = h->priv_data;
> +
> +    return ffurl_read(c->hd, buf, size);
> +}
> +
> +static int64_t abr_seek(URLContext *h, int64_t pos, int whence)
> +{
> +    ABRContext *c = h->priv_data;
> +    int64_t newpos;
> +
> +    switch (whence) {
> +    case SEEK_SET:
> +        break;
> +    case SEEK_CUR:
> +        pos = pos + c->position;
> +        break;
> +    case SEEK_END: {
unneeded braces
> +        int64_t newpos = ffurl_seek( c->hd, pos, AVSEEK_SIZE );
> +        if (newpos < 0) {
> +            av_log(h, AV_LOG_ERROR,
> +                "ABR: seek_end - can't get file size (pos=%lld)\r\n", (long long int)pos);
> +            return newpos;
> +        }
> +        pos = newpos - pos;
> +        }
unneeded braces
> +        break;
> +    case AVSEEK_SIZE: {
unneeded braces
> +        int64_t newpos = ffurl_seek( c->hd, pos, AVSEEK_SIZE );
should check newpos  value for failed.
> +        return newpos;
> +        }
unneeded braces
> +        break;
> +    default:
> +        av_log(h, AV_LOG_ERROR,
> +            "ABR: no support for seek where 'whence' is %d\r\n", whence);
> +        return AVERROR(EINVAL);
> +    }
> +
> +    newpos = ffurl_seek( c->hd, c->position, SEEK_SET );
> +    if (newpos < 0) {
> +        av_log(h, AV_LOG_ERROR,
> +            "ABR: nested protocol no support for seek or seek failed\n");
> +        return newpos;
> +    }
> +    return c->position;
> +}
> +
> +static int abr_close(URLContext *h)
> +{
> +    ABRContext *c = h->priv_data;
> +    int ret = 0;
> +
> +    ffurl_closep(&c->hd);
> +    av_free(c->variants_bitrate);
> +    av_free(c->throughputs);
> +    return ret;
> +}
> +
> +#define OFFSET(x) offsetof(ABRContext, x)
> +#define D AV_OPT_FLAG_DECODING_PARAM
> +static const AVOption options[] = {
> +    { "abr-params",  "Informations ABR needed, using a :-separated list of key=value parameters", OFFSET(abr_params), AV_OPT_TYPE_DICT, { 0 }, 0, 0, D },
> +    { "abr-metadata",  "Metadata return from abr, including switch signal and network bandwidth", OFFSET(abr_metadata), AV_OPT_TYPE_DICT, { 0 }, 0, 0, D },
> +    { NULL }
> +};
> +
> +static const AVClass abr_class = {
> +    .class_name     = "abr",
> +    .item_name      = av_default_item_name,
> +    .option         = options,
> +    .version        = LIBAVUTIL_VERSION_INT,
> +};
> +
> +const URLProtocol ff_abr_protocol = {
> +    .name                = "abr",
> +    .url_open2           = abr_open,
> +    .url_read            = abr_read,
> +    .url_seek            = abr_seek,
> +    .url_close           = abr_close,
> +    .priv_data_size      = sizeof(ABRContext),
> +    .priv_data_class     = &abr_class,
> +};
> diff --git a/libavformat/protocols.c b/libavformat/protocols.c
> index 7df18fbb3b..16c844c914 100644
> --- a/libavformat/protocols.c
> +++ b/libavformat/protocols.c
> @@ -23,6 +23,7 @@
>
>  #include "url.h"
>
> +extern const URLProtocol ff_abr_protocol;
>  extern const URLProtocol ff_async_protocol;
>  extern const URLProtocol ff_bluray_protocol;
>  extern const URLProtocol ff_cache_protocol;
> --
> 2.27.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

On Sun, Jul 05, 2020 at 19:34:54 +0800, Hongcheng Zhong wrote:
> +static int abr_rule(ABRContext *c, float bw_estimate)
> +{
> +    int ret = -1;
> +
> +    if (c->n_throughputs > 6) {
> +        if (bw_estimate < c->variants_bitrate[c->cur_pls].value / 1000 * 1.2 &&
> +            bw_estimate > c->variants_bitrate[c->cur_pls].value / 1000 * 0.8)

You are using floats, but this syntax promotes the calculation to
double before casting back to float.

Use 1.2f and 0.8f for float constants.

> +    case SEEK_END: {
> +        int64_t newpos = ffurl_seek( c->hd, pos, AVSEEK_SIZE );
> +        if (newpos < 0) {
> +            av_log(h, AV_LOG_ERROR,
> +                "ABR: seek_end - can't get file size (pos=%lld)\r\n", (long long int)pos);

Instead of casting, please just use the correct format specifier for
int64_t, which is %"PRIi64", I believe.

And no "\r\n" please - just "\n".

(Incorrect indentation as well.)

> +        }
> +        pos = newpos - pos;
> +        }
> +        break;
> +    case AVSEEK_SIZE: {
> +        int64_t newpos = ffurl_seek( c->hd, pos, AVSEEK_SIZE );
> +        return newpos;
> +        }
> +        break;
> +    default:
> +        av_log(h, AV_LOG_ERROR,
> +            "ABR: no support for seek where 'whence' is %d\r\n", whence);

Incorrect indentation.

> +        return AVERROR(EINVAL);
> +    }
> +
> +    newpos = ffurl_seek( c->hd, c->position, SEEK_SET );
> +    if (newpos < 0) {
> +        av_log(h, AV_LOG_ERROR,
> +            "ABR: nested protocol no support for seek or seek failed\n");

Incorrect indentation.

> +        return newpos;
> +    }
> +    return c->position;
> +}

I believe the blocks inside the switch/case are also incorrectly
indented.

Cheers,
Moritz

On Sun, 5 Jul 2020, Hongcheng Zhong wrote:

> From: spartazhc <spartazhc@gmail.com>
>
> Add abr module for hls/dash.
>
> Signed-off-by: spartazhc <spartazhc@gmail.com>
> ---
> doc/protocols.texi      |   7 +
> libavformat/Makefile    |   1 +
> libavformat/abr.c       | 282 ++++++++++++++++++++++++++++++++++++++++
> libavformat/protocols.c |   1 +
> 4 files changed, 291 insertions(+)
> create mode 100644 libavformat/abr.c


> +const URLProtocol ff_abr_protocol = {
> +    .name                = "abr",

If this protocol isn't a protocol that an end user would invoke (it isn't 
- there's no standard abr:// protocol), it would be good to more clearly 
mark it as an internal implementation detail instead of an end user facing 
protocol, e.g. by adding an "ff" prefix to the protocol name, like the 
existing ffrtmpcrypt protocol.

That said I haven't reviewed the concept itself though.

// Martin