From patchwork Thu Jun 11 04:43:12 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: rcombs X-Patchwork-Id: 20276 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 12E3B448CDE for ; Thu, 11 Jun 2020 07:43:45 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 0514968B3E4; Thu, 11 Jun 2020 07:43:45 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from so254-54.mailgun.net (so254-54.mailgun.net [198.61.254.54]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 202EB68B396 for ; Thu, 11 Jun 2020 07:43:37 +0300 (EEST) DKIM-Signature: a=rsa-sha256; v=1; c=relaxed/relaxed; d=rcombs.me; q=dns/txt; s=mx; t=1591850620; h=Content-Transfer-Encoding: MIME-Version: References: In-Reply-To: Message-Id: Date: Subject: To: From: Sender; bh=MqsSMmlkIhWv+YGdi8IVFCRLBwRi0fmI9764KT7fBAw=; b=wyEyMvBlZB2e1iVjZhzahOn7kDu7kofZsdoGAi6S8nUKDChahhrSNJGfEJ4cBmu+g1vPdRDY SA3hfOKvtzaJpjRRBasqYt8ysySf7pyRmixdnztrfdDuqvQNUJyVOfJJXYpsKqRYpGLGOysL cua8+7A+TS+kqvWcYOivoi163J8= X-Mailgun-Sending-Ip: 198.61.254.54 X-Mailgun-Sid: WyJiZDU1MSIsICJmZm1wZWctZGV2ZWxAZmZtcGVnLm9yZyIsICJiMGJhIl0= Received: from rcombs-mbp.localdomain ( [24.14.135.13]) by smtp-out-n12.prod.us-west-2.postgun.com with SMTP id 5ee1b6684c9690533a29659d (version=TLS1.2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256); Thu, 11 Jun 2020 04:43:20 GMT From: rcombs To: ffmpeg-devel@ffmpeg.org Date: Wed, 10 Jun 2020 23:43:12 -0500 Message-Id: <20200611044312.38981-5-rcombs@rcombs.me> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200611044312.38981-1-rcombs@rcombs.me> References: <20200611044312.38981-1-rcombs@rcombs.me> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH 5/5] lavf/dashdec: don't require opening all playlists during read_header X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" This improves startup performance massively when the consumer doesn't make a call to avformat_find_stream_info(). Also makes the requirement that each rendition only have 1 stream more clear (this is required by DASH), and generally improves error handling. --- libavformat/dashdec.c | 283 +++++++++++++++++++++++++++++++++++++++--- libavformat/version.h | 2 +- 2 files changed, 265 insertions(+), 20 deletions(-) diff --git a/libavformat/dashdec.c b/libavformat/dashdec.c index 378c892b87..b04a2596a9 100644 --- a/libavformat/dashdec.c +++ b/libavformat/dashdec.c @@ -27,6 +27,11 @@ #include "internal.h" #include "avio_internal.h" #include "dash.h" +#include "isom.h" + +#if CONFIG_H264PARSE +#include "libavcodec/h264_parse.h" +#endif #define INITIAL_BUFFER_SIZE 32768 #define MAX_MANIFEST_SIZE 50 * 1024 @@ -112,6 +117,14 @@ struct representation { int64_t cur_seg_size; struct fragment *cur_seg; + /* Media parameters */ + char *mimeType; + char *codecs; + char *audioSamplingRate; + char *sar; + char *width; + char *height; + /* Currently active Media Initialization Section */ struct fragment *init_section; uint8_t *init_sec_buf; @@ -120,6 +133,12 @@ struct representation { uint32_t init_sec_buf_read_offset; int64_t cur_timestamp; int is_restart_needed; + + int streams_initialized; + int open_failed; + + char *new_extradata; + int new_extradata_size; }; typedef struct DASHContext { @@ -152,6 +171,7 @@ typedef struct DASHContext { char *allowed_extensions; AVDictionary *avio_opts; int max_url_size; + int open_all; /* Flags for init section*/ int is_init_section_common_video; @@ -369,6 +389,11 @@ static void free_representation(struct representation *pls) } xml_freep(&pls->lang); + xml_freep(&pls->mimeType); + xml_freep(&pls->codecs); + xml_freep(&pls->audioSamplingRate); + xml_freep(&pls->width); + xml_freep(&pls->height); av_freep(&pls->url_template); av_freep(&pls); @@ -888,6 +913,16 @@ static int parse_manifest_representation(AVFormatContext *s, const char *url, } rep->lang = xmlGetProp(adaptationset_node, "lang"); +#define GET_VALUE(name) \ + if (!(rep->name = xmlGetProp(node, #name))) \ + rep->name = xmlGetProp(adaptationset_node, #name) + + GET_VALUE(mimeType); + GET_VALUE(codecs); + GET_VALUE(audioSamplingRate); + GET_VALUE(sar); + GET_VALUE(width); + GET_VALUE(height); rep->parent = s; representation_segmenttemplate_node = find_child_node_by_name(representation_node, "SegmentTemplate"); @@ -1905,7 +1940,7 @@ static int reopen_demux_for_component(AVFormatContext *s, struct representation ff_const59 AVInputFormat *in_fmt = NULL; AVDictionary *in_fmt_opts = NULL; uint8_t *avio_ctx_buffer = NULL; - int ret = 0, i; + int ret = 0; if (pls->ctx) { close_demux_for_component(pls); @@ -1969,12 +2004,14 @@ static int reopen_demux_for_component(AVFormatContext *s, struct representation av_dict_free(&in_fmt_opts); if (ret < 0) goto fail; + if (pls->ctx->nb_streams < 1) { + ret = AVERROR_INVALIDDATA; + goto fail; + } if (pls->n_fragments) { #if FF_API_R_FRAME_RATE - if (pls->framerate.den) { - for (i = 0; i < pls->ctx->nb_streams; i++) - pls->ctx->streams[i]->r_frame_rate = pls->framerate; - } + if (pls->framerate.den) + pls->ctx->streams[0]->r_frame_rate = pls->framerate; #endif ret = avformat_find_stream_info(pls->ctx, NULL); if (ret < 0) @@ -1985,10 +2022,157 @@ fail: return ret; } +static int parse_codecs(AVFormatContext *s, AVCodecParameters *params, const struct representation *pls) +{ + if (pls->mimeType && !strcmp(pls->mimeType, "text/vtt")) { + params->codec_id = AV_CODEC_ID_WEBVTT; + return 1; + } else if (pls->mimeType && !strcmp(pls->mimeType, "application/ttml+xml")) { + params->codec_id = AV_CODEC_ID_TTML; + return 1; + } else if (!pls->codecs) { + return 0; + } else if (!strncmp(pls->codecs, "avc1.", 5) || + !strncmp(pls->codecs, "avc3.", 5)) { + int len = strlen(pls->codecs); + + params->codec_id = AV_CODEC_ID_H264; + + if (len >= 11) { + char buf[3] = {0}; + char *end = NULL; + int profile_idc, level_idc; + + memcpy(buf, pls->codecs + 5, 2); + profile_idc = strtoul(buf, &end, 16); + if (end == &buf[2]) { +#if CONFIG_H264PARSE + int constraint_set_flags; + memcpy(buf, pls->codecs + 7, 2); + constraint_set_flags = strtoul(buf, &end, 16); + if (end != &buf[2]) // If this isn't a hex byte, assume no constraints + constraint_set_flags = 0; + + params->profile = avpriv_h264_get_profile(profile_idc, constraint_set_flags); +#else + params->profile = profile_idc; +#endif + } + + memcpy(buf, pls->codecs + 9, 2); + level_idc = strtoul(buf, &end, 16); + if (end == &buf[2]) + params->level = level_idc; + } + return 1; + } else if (!strncmp(pls->codecs, "hev1.", 5) || + !strncmp(pls->codecs, "hvc1.", 5)) { + unsigned profile_space = 0; + unsigned profile_idc, compat_flags; + char tier[2]; + unsigned level_idc; + unsigned extra_flags[6]; + char *str = pls->codecs + 5; + int count; + + params->codec_id = AV_CODEC_ID_H264; + + if (*str == 'A' || *str == 'B' || *str == 'C') + profile_space = (*str++) - 'A' + 1; + + if ((count = sscanf(str, "%d.%x.%[LH]%d.%x.%x.%x.%x.%x.%x", + &profile_idc, &compat_flags, tier, &level_idc, + &extra_flags[0], &extra_flags[1], &extra_flags[2], + &extra_flags[4], &extra_flags[5], &extra_flags[6])) >= 4) { + params->profile = profile_idc; + params->level = level_idc; + } + + return 1; + } else if (!strncmp(pls->codecs, "mp4a.", 5)) { + int count; + unsigned oui, aot; + + if ((count = sscanf(pls->codecs + 5, "%x.%d", &oui, &aot)) < 1) + return 0; + + if (!(params->codec_id = ff_codec_get_id(ff_mp4_obj_type, oui))) + return 0; + + if (count == 2 && params->codec_id == AV_CODEC_ID_AAC && aot > 0) + params->profile = aot - 1; + + return 1; + } else if (!strcmp(pls->codecs, "ac-3")) { + params->codec_id = AV_CODEC_ID_AC3; + return 1; + } else if (!strcmp(pls->codecs, "ec-3")) { + params->codec_id = AV_CODEC_ID_EAC3; + return 1; + } else if (!strcmp(pls->codecs, "mlpa")) { + params->codec_id = AV_CODEC_ID_TRUEHD; + return 1; + } else if (!strcmp(pls->codecs, "dtsc")) { + params->codec_id = AV_CODEC_ID_DTS; + params->profile = FF_PROFILE_DTS; + return 1; + } else if (!strcmp(pls->codecs, "dtsh")) { + params->codec_id = AV_CODEC_ID_DTS; + params->profile = FF_PROFILE_DTS_HD_HRA; // Can't distinguish HRA from MA here + return 1; + } else if (!strcmp(pls->codecs, "dtse")) { + params->codec_id = AV_CODEC_ID_DTS; + params->profile = FF_PROFILE_DTS_EXPRESS; + return 1; + } else if (!strcmp(pls->codecs, "dtsl")) { + params->codec_id = AV_CODEC_ID_DTS; + params->profile = FF_PROFILE_DTS_HD_MA; // No-core stream + return 1; + } else if (!strncmp(pls->codecs, "mhm1.", 5)) { + params->codec_id = AV_CODEC_ID_MPEGH_3D_AUDIO; // There's also a profile here but we can't express it yet + return 1; + } else if (!strcmp(pls->codecs, "vp8") || + !strncmp(pls->codecs, "vp8.", 4)) { + params->codec_id = AV_CODEC_ID_VP8; // Optional profile not parsed + return 1; + } else if (!strcmp(pls->codecs, "vp9") || + !strncmp(pls->codecs, "vp9.", 4)) { + params->codec_id = AV_CODEC_ID_VP8; // Optional profile not parsed + return 1; + } else if (!strncmp(pls->codecs, "av01.", 5)) { + params->codec_id = AV_CODEC_ID_AV1; // Profile/level data not parsed + return 1; + } else { + return 0; + } +} + +static int update_parameters(struct representation *pls, AVStream *st, int extern_extradata) +{ + int ret; + AVStream *ist = pls->ctx->streams[0]; + if ((ret = avcodec_parameters_copy(st->codecpar, ist->codecpar)) < 0) + return ret; + + avpriv_set_pts_info(st, ist->pts_wrap_bits, ist->time_base.num, ist->time_base.den); + st->internal->need_context_update = 1; + + if (extern_extradata && st->codecpar->extradata) { + pls->new_extradata = st->codecpar->extradata; + pls->new_extradata_size = st->codecpar->extradata_size; + st->codecpar->extradata = NULL; + st->codecpar->extradata_size = 0; + } + + return 0; +} + static int open_demux_for_component(AVFormatContext *s, struct representation *pls) { + DASHContext *c = s->priv_data; int ret = 0; - int i; + int gotFullParams = 0; + AVCodecParameters *params = NULL; pls->parent = s; pls->cur_seq_no = calc_cur_seg_no(s, pls); @@ -1997,24 +2181,60 @@ static int open_demux_for_component(AVFormatContext *s, struct representation *p pls->last_seq_no = calc_max_seg_no(pls, s->priv_data); } - ret = reopen_demux_for_component(s, pls); - if (ret < 0) { - goto fail; + if (!c->open_all) { + if (!(params = avcodec_parameters_alloc())) { + ret = AVERROR(ENOMEM); + goto fail; + } + params->codec_type = pls->type; + if (parse_codecs(s, params, pls)) { + if (pls->type == AVMEDIA_TYPE_VIDEO) { + if (pls->width) + params->width = atoi(pls->width); + if (pls->height) + params->height = atoi(pls->height); + if (pls->sar) + av_parse_ratio_quiet(¶ms->sample_aspect_ratio, pls->sar, 1001000); + gotFullParams = (pls->width > 0) && (pls->height > 0); + } else if (pls->type == AVMEDIA_TYPE_AUDIO) { + if (pls->audioSamplingRate) + params->sample_rate = atoi(pls->audioSamplingRate); + gotFullParams = (params->sample_rate > 0); + } else if (pls->type == AVMEDIA_TYPE_SUBTITLE) { + gotFullParams = 1; + } + } } - for (i = 0; i < pls->ctx->nb_streams; i++) { + + if (gotFullParams) { AVStream *st = avformat_new_stream(s, NULL); - AVStream *ist = pls->ctx->streams[i]; if (!st) { ret = AVERROR(ENOMEM); goto fail; } - st->id = i; - avcodec_parameters_copy(st->codecpar, ist->codecpar); - avpriv_set_pts_info(st, ist->pts_wrap_bits, ist->time_base.num, ist->time_base.den); + st->id = 0; + avcodec_parameters_copy(st->codecpar, params); + } else { + AVStream *st; + ret = reopen_demux_for_component(s, pls); + if (ret < 0) { + goto fail; + } + if (!(st = avformat_new_stream(s, NULL))) { + ret = AVERROR(ENOMEM); + goto fail; + } + st->id = 0; + if ((ret = update_parameters(pls, st, 0)) < 0) + goto fail; + + pls->streams_initialized = 1; } - return 0; + ret = 0; + fail: + avcodec_parameters_free(¶ms); return ret; } @@ -2200,15 +2420,24 @@ static void recheck_discard_flags(AVFormatContext *s, struct representation **p, struct representation *pls = p[i]; int needed = !pls->assoc_stream || pls->assoc_stream->discard < AVDISCARD_ALL; - if (needed && !pls->ctx) { + if (needed && !pls->ctx && !pls->open_failed) { pls->cur_seg_offset = 0; pls->init_sec_buf_read_offset = 0; /* Catch up */ for (j = 0; j < n; j++) { pls->cur_seq_no = FFMAX(pls->cur_seq_no, p[j]->cur_seq_no); } - reopen_demux_for_component(s, pls); - av_log(s, AV_LOG_INFO, "Now receiving stream_index %d\n", pls->stream_index); + if (reopen_demux_for_component(s, pls) >= 0) { + av_log(s, AV_LOG_INFO, "Now receiving stream_index %d\n", pls->stream_index); + + if (!pls->streams_initialized) { + AVStream *st = s->streams[pls->stream_index]; + update_parameters(pls, st, 1); + pls->streams_initialized = 1; + } + } else { + pls->open_failed = 1; + } } else if (!needed && pls->ctx) { close_demux_for_component(pls); ff_format_io_close(pls->parent, &pls->input); @@ -2267,7 +2496,20 @@ static int dash_read_packet(AVFormatContext *s, AVPacket *pkt) /* If we got a packet, return it */ cur->cur_timestamp = av_rescale(pkt->pts, (int64_t)cur->ctx->streams[0]->time_base.num * 90000, cur->ctx->streams[0]->time_base.den); pkt->stream_index = cur->stream_index; - return 0; + + if (cur->new_extradata) { + ret = av_packet_add_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, + cur->new_extradata, cur->new_extradata_size); + if (ret >= 0) { + cur->new_extradata = NULL; + cur->new_extradata_size = 0; + } + } + + if (ret >= 0) + return 0; + else + av_packet_unref(pkt); } if (cur->is_restart_needed) { cur->cur_seg_offset = 0; @@ -2409,6 +2651,9 @@ static const AVOption dash_options[] = { OFFSET(allowed_extensions), AV_OPT_TYPE_STRING, {.str = "aac,m4a,m4s,m4v,mov,mp4,webm,ts"}, INT_MIN, INT_MAX, FLAGS}, + {"open_all", "Whether to read all variant headers during startup", + OFFSET(open_all), AV_OPT_TYPE_BOOL, {.i64 = 0}, + 0, 1, FLAGS}, {NULL} }; diff --git a/libavformat/version.h b/libavformat/version.h index 59151a71a0..ea2ccb400c 100644 --- a/libavformat/version.h +++ b/libavformat/version.h @@ -33,7 +33,7 @@ // Also please add any ticket numbers that you believe might be affected here #define LIBAVFORMAT_VERSION_MAJOR 58 #define LIBAVFORMAT_VERSION_MINOR 46 -#define LIBAVFORMAT_VERSION_MICRO 101 +#define LIBAVFORMAT_VERSION_MICRO 102 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \ LIBAVFORMAT_VERSION_MINOR, \