[FFmpeg-devel,RFC,GSoC,v1,3/6] avformat/hls: use abr to switch streams

From: spartazhc <spartazhc@gmail.com>

When abr is enable, it will take over the task to call http to
download segments, and will return a switch-request for hls to
switch streams.
For reason not to waste segments that have been downloaded,
switch will become effective after old segments is used out.
Abr cannot work with http_persistent option, and currently use
http_multiple.

v1 fixed:
1. fix memory leak

Signed-off-by: spartazhc <spartazhc@gmail.com>
---
 doc/demuxers.texi |   3 +
 libavformat/hls.c | 232 ++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 229 insertions(+), 6 deletions(-)

Hongcheng Zhong <sj.hc_Zhong@sjtu.edu.cn> 于2020年7月15日周三 下午4:38写道：
>
> From: spartazhc <spartazhc@gmail.com>
>
> When abr is enable, it will take over the task to call http to
> download segments, and will return a switch-request for hls to
> switch streams.
> For reason not to waste segments that have been downloaded,
> switch will become effective after old segments is used out.
> Abr cannot work with http_persistent option, and currently use
> http_multiple.
>
> v1 fixed:
> 1. fix memory leak
>
> Signed-off-by: spartazhc <spartazhc@gmail.com>
> ---
>  doc/demuxers.texi |   3 +
>  libavformat/hls.c | 232 ++++++++++++++++++++++++++++++++++++++++++++--
>  2 files changed, 229 insertions(+), 6 deletions(-)
>
> diff --git a/doc/demuxers.texi b/doc/demuxers.texi
> index 3c15ab9eee..4cdbd95962 100644
> --- a/doc/demuxers.texi
> +++ b/doc/demuxers.texi
> @@ -321,6 +321,9 @@ available in a metadata key named "variant_bitrate".
>  It accepts the following options:
>
>  @table @option
> +@item abr
> +enable abr to switch streams.
> +
>  @item live_start_index
>  segment index to start live streams at (negative values are from the end).
>
> diff --git a/libavformat/hls.c b/libavformat/hls.c
> index ba17c4ed96..877bd8c677 100644
> --- a/libavformat/hls.c
> +++ b/libavformat/hls.c
> @@ -189,6 +189,17 @@ struct variant {
>      char subtitles_group[MAX_FIELD_LEN];
>  };
>
> +struct throughput {
> +    int n_throughputs;
> +
> +    /* throughputs are in kbps, always record the last 20 times
> +     * set record 20 times refer to paper FESTIVE https://doi.org/10.1145/2413176.2413189
> +     */
> +    float throughput_fifo[20];
> +    int head;
> +    int tail;
> +};

what about add an #define blablabla_name 20 here if use the record 20
times, and use the blablabla_name at the below code which use the 20
value?
> +
>  typedef struct HLSContext {
>      AVClass *class;
>      AVFormatContext *ctx;
> @@ -213,8 +224,36 @@ typedef struct HLSContext {
>      int http_multiple;
>      int http_seekable;
>      AVIOContext *playlist_pb;
> +
> +    int abr;
> +    struct throughput *throughputs;
> +    int can_switch;
> +    int switch_request2;
> +    int switch_delay;
> +    int64_t switch_timestamp;
> +    int64_t delta_timestamp;
> +    int cur_pls;
>  } HLSContext;
>
> +static struct segment *next_segment(struct playlist *pls);
> +static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg, AVIOContext **in);
> +
> +static void sync_cur_seq(HLSContext *c) {
> +    int i;
> +    for (i = 0; i < c->n_playlists; i++) {
> +        struct playlist *pls = c->playlists[i];
> +        pls->cur_seq_no = c->cur_seq_no;
> +    }
> +}
> +
> +static struct segment *next2_segment(struct playlist *pls)
> +{
> +    int n = pls->cur_seq_no - pls->start_seq_no + 2;
> +    if (n >= pls->n_segments)
> +        return NULL;
> +    return pls->segments[n];
> +}
> +
>  static void free_segment_dynarray(struct segment **segments, int n_segments)
>  {
>      int i;
> @@ -624,6 +663,31 @@ static int open_url_keepalive(AVFormatContext *s, AVIOContext **pb,
>  #endif
>  }
>
> +static int update_throughputs(struct throughput *thr, float time, int pb_size)
> +{
> +    if (pb_size <= 0 || time <= 0)
> +        return -1;
EINVAL?
> +    if (thr->n_throughputs < 20) {
> +        ++thr->n_throughputs;
> +    } else {
> +        ++thr->head;
> +    }
> +    thr->throughput_fifo[thr->tail] = (float)(pb_size) / time;
> +    thr->tail = (thr->tail + 1) % 20;
> +    return 0;
> +}
> +
> +static int64_t get_switch_timestamp(HLSContext *c, struct playlist *pls)
> +{
> +    int64_t pos = c->first_timestamp == AV_NOPTS_VALUE ?
> +                  0 : c->first_timestamp;
> +
> +    for (int i = 0; i < pls->cur_seq_no + 2; i++) {
> +        pos += pls->segments[i]->duration;
> +    }
> +    return pos;
> +}
> +
>  static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>                      AVDictionary *opts, AVDictionary *opts2, int *is_http_out)
>  {
> @@ -639,6 +703,9 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>      } else if (av_strstart(url, "data", NULL)) {
>          if (url[4] == '+' || url[4] == ':')
>              proto_name = avio_find_protocol_name(url + 5);
> +    } else if (av_strstart(url, "ffabr", NULL)) {
> +        if (url[5] == '+' || url[5] == ':')
> +            proto_name = avio_find_protocol_name(url + 6);
>      }
>
>      if (!proto_name)
> @@ -669,6 +736,8 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>          ;
>      else if (av_strstart(url, "data", NULL) && !strncmp(proto_name, url + 5, strlen(proto_name)) && url[5 + strlen(proto_name)] == ':')
>          ;
> +    else if (av_strstart(url, "ffabr", NULL) && !strncmp(proto_name, url + 6, strlen(proto_name)) && url[6 + strlen(proto_name)] == ':')
> +        ;
>      else if (strcmp(proto_name, "file") || !strncmp(url, "file,", 5))
>          return AVERROR_INVALIDDATA;
>
> @@ -690,6 +759,43 @@ static int open_url(AVFormatContext *s, AVIOContext **pb, const char *url,
>      } else {
>          ret = s->io_open(s, pb, url, AVIO_FLAG_READ, &tmp);
>      }
> +    if (c->abr && ret >= 0) {
> +        AVDictionary *abr_ret = NULL;
> +        AVDictionaryEntry *en = NULL;
> +        struct segment *seg;
> +        int pb_size, switch_request;
> +        av_opt_get_dict_val(*pb, "abr-metadata", AV_OPT_SEARCH_CHILDREN, &abr_ret);
> +        if (abr_ret) {
> +            en = av_dict_get(abr_ret, "download_time", NULL, 0);
> +            if (en) {
> +                pb_size = avio_size(*pb);
> +                update_throughputs(c->throughputs, atoi(en->value) / 1000.0, pb_size);
> +                av_log(s, AV_LOG_VERBOSE, "[abr] time=%.2fms, size=%.2fkb\n", atoi(en->value) / 1000.0, pb_size / 1000.0);
> +            }
> +            en = av_dict_get(abr_ret, "switch_request", NULL, 0);
> +            if (en) {
> +                switch_request = atoi(en->value);
> +                av_log(s, AV_LOG_VERBOSE, "[abr] switch request: %s\n", en->value);
> +            }
> +            if (switch_request != -1) {
> +                c->switch_request2 = switch_request;
> +                c->switch_delay = 2;
> +                c->can_switch = 0;
> +                sync_cur_seq(c);
> +                seg = next2_segment(c->playlists[c->switch_request2]);
> +                if (!seg) {
> +                    c->switch_request2 = -1;
> +                    av_log(s, AV_LOG_INFO, "[abr] no more segment need to switch\n");
> +                } else {
> +                    c->switch_timestamp = get_switch_timestamp(c,  c->playlists[c->switch_request2]);
> +                    av_log(s, AV_LOG_VERBOSE, "[abr] switch timestamp: %ld\n", c->switch_timestamp);
> +                    c->playlists[c->switch_request2]->input_next_requested = 1;
> +                    ret = open_input(c, c->playlists[c->switch_request2], seg, &c->playlists[c->switch_request2]->input_next);
> +                }
> +            }
> +            av_dict_free(&abr_ret);
> +        }
> +    }
>      if (ret >= 0) {
>          // update cookies on http response with setcookies.
>          char *new_cookies = NULL;
> @@ -1219,6 +1325,33 @@ static void intercept_id3(struct playlist *pls, uint8_t *buf,
>          pls->is_id3_timestamped = (pls->id3_mpegts_timestamp != AV_NOPTS_VALUE);
>  }
>
> +static int abrinfo_to_dict(HLSContext *c, char **abr_info)
> +{
> +    struct throughput *thr = c->throughputs;
> +    char buffer[MAX_URL_SIZE];
> +    int size, i;
> +    size = snprintf(buffer, sizeof(buffer), "format=hls:");
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "cur_pls=%d:", c->cur_pls);
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "can_switch=%d:", c->can_switch);
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "n_variants=%d:", c->n_variants);
> +    for (i = 0; i < c->n_variants; i++) {
> +        struct variant *v = c->variants[i];
> +        size += snprintf(buffer + size, sizeof(buffer) - size, "variant_bitrate%d=%d:", i, v->bandwidth);
> +    }
> +    size += snprintf(buffer + size, sizeof(buffer) - size, "n_throughputs=%d:", thr->n_throughputs);
> +    if (thr->n_throughputs > 0) {
> +        i = thr->head;
> +        do {
> +            size += snprintf(buffer + size, sizeof(buffer) - size, "throughputs%d=%.2f:", i, thr->throughput_fifo[i]);
> +            i = (i + 1) % 20;
> +        } while (i != thr->tail);
> +    }
> +    *abr_info = av_malloc(size);
check the return NULL of av_malloc.
> +    snprintf(*abr_info, size, "%s", buffer);
> +    av_log(c, AV_LOG_DEBUG, "[abr] abr_info: %s\n", *abr_info);
> +    return 0;
> +}
> +
>  static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg, AVIOContext **in)
>  {
>      AVDictionary *opts = NULL;
> @@ -1238,8 +1371,20 @@ static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg,
>      av_log(pls->parent, AV_LOG_VERBOSE, "HLS request for url '%s', offset %"PRId64", playlist %d\n",
>             seg->url, seg->url_offset, pls->index);
>
> +    if (c->abr) {
> +        char *abr_opts;
> +        abrinfo_to_dict(c, &abr_opts);
> +        av_dict_set(&opts, "abr-params", abr_opts, 0); // how to setup flag?
> +        av_free(abr_opts);
> +    }
>      if (seg->key_type == KEY_NONE) {
> -        ret = open_url(pls->parent, in, seg->url, c->avio_opts, opts, &is_http);
> +        char url_abr[MAX_URL_SIZE];
> +        if (c->abr) {
> +            snprintf(url_abr, sizeof(url_abr), "ffabr:%s", seg->url); // + or : tbd
> +            ret = open_url(pls->parent, in, url_abr, c->avio_opts, opts, &is_http);
> +        } else {
> +            ret = open_url(pls->parent, in, seg->url, c->avio_opts, opts, &is_http);
> +        }
>      } else if (seg->key_type == KEY_AES_128) {
>          char iv[33], key[33], url[MAX_URL_SIZE];
>          if (strcmp(seg->key, pls->key_url)) {
> @@ -1273,6 +1418,7 @@ static int open_input(HLSContext *c, struct playlist *pls, struct segment *seg,
>              goto cleanup;
>          }
>          ret = 0;
> +        // TODO: Add abr prefix for crypto
>      } else if (seg->key_type == KEY_SAMPLE_AES) {
>          av_log(pls->parent, AV_LOG_ERROR,
>                 "SAMPLE-AES encryption is not supported yet\n");
> @@ -1435,7 +1581,17 @@ restart:
>          /* Check that the playlist is still needed before opening a new
>           * segment. */
>          v->needed = playlist_needed(v);
> -
> +        if (c->abr) {
> +            if (c->switch_request2 == -1)
> +                ;
> +            else if (v->needed && c->switch_request2 != v->index && c->switch_delay <= 0) {
> +                av_log(v->parent, AV_LOG_VERBOSE, "read_data: needed but not download playlist %d ('%s')\n",
> +                       v->index, v->url);
> +                return AVERROR_EOF;
> +            }
> +            if (!c->can_switch)
> +                c->can_switch = 1;
> +        }
>          if (!v->needed) {
>              av_log(v->parent, AV_LOG_INFO, "No longer receiving playlist %d ('%s')\n",
>                     v->index, v->url);
> @@ -1530,7 +1686,12 @@ reload:
>      }
>
>      seg = next_segment(v);
> -    if (c->http_multiple == 1 && !v->input_next_requested &&
> +
> +    // when get switch_request, old stream should stop downloading next seg
> +    if (c->abr && c->switch_request2 != v->index && c->switch_timestamp != AV_NOPTS_VALUE
> +         && ((c->cur_timestamp + seg->duration * 2) >= c->switch_timestamp) )
> +        ;
> +    else if (c->http_multiple == 1 && !v->input_next_requested &&
else if (c->http_multiple && !v->input_next_requested &&
>          seg && seg->key_type == KEY_NONE && av_strstart(seg->url, "http", NULL)) {
>          ret = open_input(c, v, seg, &v->input_next);
>          if (ret < 0) {
> @@ -1563,11 +1724,17 @@ reload:
>
>          return ret;
>      }
> -    if (c->http_persistent &&
> +    if (c->http_persistent && !c->abr &&
>          seg->key_type == KEY_NONE && av_strstart(seg->url, "http", NULL)) {
>          v->input_read_done = 1;
>      } else {
>          ff_format_io_close(v->parent, &v->input);
> +        if (c->abr) {
> +            if (c->switch_delay > 0)
> +                c->switch_delay--;
> +            av_log(v->parent, AV_LOG_VERBOSE, "read_data: close pls[%d]->input, v->cur_seq_no=%d, c->switch_delay= %d\n",
> +                    v->index, v->cur_seq_no, c->switch_delay);
> +        }
>      }
>      v->cur_seq_no++;
>
> @@ -1860,6 +2027,17 @@ static int hls_read_header(AVFormatContext *s)
>      c->first_packet = 1;
>      c->first_timestamp = AV_NOPTS_VALUE;
>      c->cur_timestamp = AV_NOPTS_VALUE;
> +    c->switch_request2 = -1;
> +    c->switch_delay = 0;
> +    c->switch_timestamp = AV_NOPTS_VALUE;
> +    c->delta_timestamp = AV_NOPTS_VALUE;
> +    c->can_switch = -1;
> +
> +    if (c->abr) {
> +        c->http_persistent = 0;
> +        c->http_multiple = 1;
> +        c->throughputs = av_mallocz(sizeof(struct throughput));
check return NULL;
> +    }
>
>      if ((ret = save_avio_options(s)) < 0)
>          goto fail;
> @@ -2049,6 +2227,9 @@ static int hls_read_header(AVFormatContext *s)
>          add_metadata_from_renditions(s, pls, AVMEDIA_TYPE_SUBTITLE);
>      }
>
> +    if (c->abr && c->can_switch == -1)
> +        c->can_switch = 1;
> +
>      update_noheader_flag(s);
>
>      return 0;
> @@ -2057,6 +2238,13 @@ fail:
>      return ret;
>  }
>
> +static void change_discard_flags(struct playlist *pls, int flag)
> +{
> +    for (int i = 0; i < pls->n_main_streams; i++) {
> +        pls->main_streams[i]->discard = flag;
> +    }
> +}
> +
>  static int recheck_discard_flags(AVFormatContext *s, int first)
>  {
>      HLSContext *c = s->priv_data;
> @@ -2067,6 +2255,22 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>      for (i = 0; i < c->n_playlists; i++) {
>          struct playlist *pls = c->playlists[i];
>
> +        if (c->abr) {
> +            if (c->switch_request2 == -1)
> +                ;
> +            else if (c->switch_request2 == i && c->switch_delay <= 0
> +                    && c->cur_timestamp + c->delta_timestamp >= c->switch_timestamp
> +                    && c->switch_timestamp != AV_NOPTS_VALUE) {
> +                av_log(s, AV_LOG_VERBOSE, "[switch point] cur_timestamp:%ld\n", c->cur_timestamp);
> +                change_discard_flags(pls, AVDISCARD_DEFAULT);
> +                c->switch_timestamp = AV_NOPTS_VALUE;
> +            } else if (c->switch_request2 != -1 && c->switch_request2 != i && c->switch_delay <= 0
> +                    && c->cur_timestamp + c->delta_timestamp >= c->switch_timestamp
> +                    && c->switch_timestamp != AV_NOPTS_VALUE) {
> +                change_discard_flags(pls, AVDISCARD_ALL);
> +            }
> +        }
> +
>          cur_needed = playlist_needed(c->playlists[i]);
>
>          if (pls->broken) {
> @@ -2077,6 +2281,11 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>              changed = 1;
>              pls->cur_seq_no = select_cur_seq_no(c, pls);
>              pls->pb.eof_reached = 0;
> +            if (c->abr) {
> +                pls->cur_seq_no = select_cur_seq_no(c, pls) + 1;
> +                avio_flush(&pls->pb);
> +                avformat_flush(pls->ctx);
> +            }
>              if (c->cur_timestamp != AV_NOPTS_VALUE) {
>                  /* catch up */
>                  pls->seek_timestamp = c->cur_timestamp;
> @@ -2084,7 +2293,7 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>                  pls->seek_stream_index = -1;
>              }
>              av_log(s, AV_LOG_INFO, "Now receiving playlist %d, segment %d\n", i, pls->cur_seq_no);
> -        } else if (first && !cur_needed && pls->needed) {
> +        } else if ((first || c->abr) && !cur_needed && pls->needed) {
>              ff_format_io_close(pls->parent, &pls->input);
>              pls->input_read_done = 0;
>              ff_format_io_close(pls->parent, &pls->input_next);
> @@ -2093,6 +2302,9 @@ static int recheck_discard_flags(AVFormatContext *s, int first)
>              changed = 1;
>              av_log(s, AV_LOG_INFO, "No longer receiving playlist %d\n", i);
>          }
> +
> +        if (c->abr && changed && cur_needed)
> +            c->cur_pls = i;
>      }
>      return changed;
>  }
> @@ -2167,7 +2379,13 @@ static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
>                          /* audio elementary streams are id3 timestamped */
>                          fill_timing_for_id3_timestamped_stream(pls);
>                      }
> -
> +                    if (c->abr && c->first_timestamp != AV_NOPTS_VALUE && c->delta_timestamp == AV_NOPTS_VALUE && !i) {
> +                        c->delta_timestamp = av_rescale_q(pls->pkt.dts,
> +                            get_timebase(pls), AV_TIME_BASE_Q) - c->first_timestamp + 1;
> +                        av_log(s, AV_LOG_VERBOSE, "[abr] delta_timestamp=%ld, %ld, %ld\n",
> +                             c->delta_timestamp, c->first_timestamp,av_rescale_q(pls->pkt.dts,
> +                            get_timebase(pls), AV_TIME_BASE_Q));
> +                    }
>                      if (c->first_timestamp == AV_NOPTS_VALUE &&
>                          pls->pkt.dts       != AV_NOPTS_VALUE)
>                          c->first_timestamp = av_rescale_q(pls->pkt.dts,
> @@ -2375,6 +2593,8 @@ static int hls_probe(const AVProbeData *p)
>  #define OFFSET(x) offsetof(HLSContext, x)
>  #define FLAGS AV_OPT_FLAG_DECODING_PARAM
>  static const AVOption hls_options[] = {
> +    {"abr", "enable abr to switch streams",
> +        OFFSET(abr), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, FLAGS },
>      {"live_start_index", "segment index to start live streams at (negative values are from the end)",
>          OFFSET(live_start_index), AV_OPT_TYPE_INT, {.i64 = -3}, INT_MIN, INT_MAX, FLAGS},
>      {"allowed_extensions", "List of file extensions that hls is allowed to access",
> --
> 2.27.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".