[FFmpeg-devel] avformat/hls: add supporting fMP4(CMAF) format to seek on HLS demuxer

Signed-off-by: Dongwon Kim <dongwon.kim@sk.com>
---
 libavformat/hls.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

             pls->seek_flags |= AVSEEK_FLAG_ANY;
         }
+
+        if (pls->ctx && pls->ctx->iformat && strcmp(pls->ctx->iformat->name, "mov,mp4,m4a,3gp,3g2,mj2") == 0) {
+            pls->init_sec_buf_read_offset = 0;
+            reopen_demux_for_component(s, pls);
+        }
     }
 
     c->cur_timestamp = seek_timestamp;
--
2.25.1

HLS spec RFC 8216(https://tools.ietf.org/html/rfc8216) version 7 added supporting fMP4(CMAF) format for segment. However, when requesting seek operation,
the hls demuxer doesn't work properly due to previous implementation of the HLS demuxer was only for supporting MPEG-TS format for segment.

So, I added procedure to reopen segment format on HLS demuxer when seeking requested. It's quite similar to MPEG-DASH demuxer(libavformat/dashdec.c) seeking procedure, as you know, MPEG-DASH also support fMP4(CMAF) format for fragment as default. Refer to https://github.com/FFmpeg/FFmpeg/blob/master/libavformat/dashdec.c#L1904.


Please test with below command:

Server:
$ ffmpeg -y -i {input} -vcodec libx264 \
    -force_key_frames 30 \
    -sc_threshold 0 \
    -acodec aac \
    -hls_segment_type fmp4 \
    -hls_time 6 \
    -hls_playlist_type vod \
    -map 0:a -map 0:v -f hls \
    -var_stream_map "a:0,agroup:aud,default:yes v:0,agroup:aud" \
    -hls_segment_filename "segment_%v_%03d.m4s" \
    -master_pl_name playlist.m3u8 \
    hlscmaf_%v.m3u8


Player:
$ ffplay playlist.m3u8



On 20. 7. 15. 오후 4:20, Dongwon Kim wrote:
> Signed-off-by: Dongwon Kim <dongwon.kim@sk.com>
> ---
>   libavformat/hls.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 67 insertions(+)
>
> diff --git a/libavformat/hls.c b/libavformat/hls.c
> index ba17c4ed96..561b42ea6b 100644
> --- a/libavformat/hls.c
> +++ b/libavformat/hls.c
> @@ -2139,6 +2139,68 @@ static int compare_ts_with_wrapdetect(int64_t ts_a, struct playlist *pls_a,
>       return av_compare_mod(scaled_ts_a, scaled_ts_b, 1LL << 33);
>   }
>   +static int reopen_demux_for_component(AVFormatContext *s, struct playlist *pls)
> +{
> +    ff_const59 AVInputFormat *in_fmt = NULL;
> +    AVDictionary  *in_fmt_opts = NULL;
> +    uint8_t *avio_ctx_buffer  = NULL;
> +    int ret = 0;
> +
> +    if (pls->ctx) {
> +        av_freep(&pls->pb.buffer);
> +        memset(&pls->pb, 0x00, sizeof(AVIOContext));
> +        pls->ctx->pb = NULL;
> +        avformat_close_input(&pls->ctx);
> +        pls->ctx = NULL;
> +    }
> +
> +    if (ff_check_interrupt(&s->interrupt_callback)) {
> +        ret = AVERROR_EXIT;
> +        goto fail;
> +    }
> +
> +    if (!(pls->ctx = avformat_alloc_context())) {
> +        ret = AVERROR(ENOMEM);
> +        goto fail;
> +    }
> +
> +    avio_ctx_buffer  = av_malloc(INITIAL_BUFFER_SIZE);
> +    if (!avio_ctx_buffer ) {
> +        ret = AVERROR(ENOMEM);
> +        avformat_free_context(pls->ctx);
> +        pls->ctx = NULL;
> +        goto fail;
> +    }
> +    ffio_init_context(&pls->pb, avio_ctx_buffer , INITIAL_BUFFER_SIZE, 0, pls, read_data, NULL, NULL);
> +    pls->pb.seekable = 0;
> +
> +    if ((ret = ff_copy_whiteblacklists(pls->ctx, s)) < 0)
> +        goto fail;
> +
> +    pls->ctx->flags = AVFMT_FLAG_CUSTOM_IO;
> +    ret = av_probe_input_buffer(&pls->pb, &in_fmt, "", NULL, 0, 0);
> +    if (ret < 0) {
> +        av_log(s, AV_LOG_ERROR, "Error when loading segment, playlist %d\n", pls->cur_seq_no);
> +        avformat_free_context(pls->ctx);
> +        pls->ctx = NULL;
> +        goto fail;
> +    }
> +
> +    pls->ctx->pb = &pls->pb;
> +    pls->ctx->io_open  = nested_io_open;
> +
> +    ret = avformat_open_input(&pls->ctx, "", in_fmt, &in_fmt_opts);
> +    av_dict_free(&in_fmt_opts);
> +    if (ret < 0)
> +        goto fail;
> +    ret = avformat_find_stream_info(pls->ctx, NULL);
> +    if (ret < 0)
> +        goto fail;
> +
> +fail:
> +    return ret;
> +}
> +
>   static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
>   {
>       HLSContext *c = s->priv_data;
> @@ -2351,6 +2413,11 @@ static int hls_read_seek(AVFormatContext *s, int stream_index,
>               pls->seek_stream_index = -1;
>               pls->seek_flags |= AVSEEK_FLAG_ANY;
>           }
> +
> +        if (pls->ctx && pls->ctx->iformat && strcmp(pls->ctx->iformat->name, "mov,mp4,m4a,3gp,3g2,mj2") == 0) {
> +            pls->init_sec_buf_read_offset = 0;
> +            reopen_demux_for_component(s, pls);
> +        }
>       }
>         c->cur_timestamp = seek_timestamp;

> On 16 Jul 2020, at 10:36, Dongwon Kim <dongwon00.kim@gmail.com> wrote:
> 
> 
> HLS spec RFC 8216(https://tools.ietf.org/html/rfc8216) version 7 added supporting fMP4(CMAF) format for segment. However, when requesting seek operation,
> the hls demuxer doesn't work properly due to previous implementation of the HLS demuxer was only for supporting MPEG-TS format for segment.
> 
> So, I added procedure to reopen segment format on HLS demuxer when seeking requested. It's quite similar to MPEG-DASH demuxer(libavformat/dashdec.c) seeking procedure, as you know, MPEG-DASH also support fMP4(CMAF) format for fragment as default. Refer to https://github.com/FFmpeg/FFmpeg/blob/master/libavformat/dashdec.c#L1904.
> 
> 
> Please test with below command:
> 
> Server:
> $ ffmpeg -y -i {input} -vcodec libx264 \
>    -force_key_frames 30 \
>    -sc_threshold 0 \
>    -acodec aac \
>    -hls_segment_type fmp4 \
>    -hls_time 6 \
>    -hls_playlist_type vod \
>    -map 0:a -map 0:v -f hls \
>    -var_stream_map "a:0,agroup:aud,default:yes v:0,agroup:aud" \
>    -hls_segment_filename "segment_%v_%03d.m4s" \
>    -master_pl_name playlist.m3u8 \
>    hlscmaf_%v.m3u8
> 
> 
> Player:
> $ ffplay playlist.m3u8
> 
> 
> 
> On 20. 7. 15. 오후 4:20, Dongwon Kim wrote:
>> Signed-off-by: Dongwon Kim <dongwon.kim@sk.com>
>> ---
>>  libavformat/hls.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++
>>  1 file changed, 67 insertions(+)
>> 
>> diff --git a/libavformat/hls.c b/libavformat/hls.c
>> index ba17c4ed96..561b42ea6b 100644
>> --- a/libavformat/hls.c
>> +++ b/libavformat/hls.c
>> @@ -2139,6 +2139,68 @@ static int compare_ts_with_wrapdetect(int64_t ts_a, struct playlist *pls_a,
>>      return av_compare_mod(scaled_ts_a, scaled_ts_b, 1LL << 33);
>>  }
>>  +static int reopen_demux_for_component(AVFormatContext *s, struct playlist *pls)
>> +{
>> +    ff_const59 AVInputFormat *in_fmt = NULL;
>> +    AVDictionary  *in_fmt_opts = NULL;
>> +    uint8_t *avio_ctx_buffer  = NULL;
>> +    int ret = 0;
>> +
>> +    if (pls->ctx) {
>> +        av_freep(&pls->pb.buffer);
>> +        memset(&pls->pb, 0x00, sizeof(AVIOContext));
>> +        pls->ctx->pb = NULL;
>> +        avformat_close_input(&pls->ctx);
>> +        pls->ctx = NULL;
>> +    }
>> +
>> +    if (ff_check_interrupt(&s->interrupt_callback)) {
>> +        ret = AVERROR_EXIT;
>> +        goto fail;
>> +    }
>> +
>> +    if (!(pls->ctx = avformat_alloc_context())) {
>> +        ret = AVERROR(ENOMEM);
>> +        goto fail;
>> +    }
>> +
>> +    avio_ctx_buffer  = av_malloc(INITIAL_BUFFER_SIZE);
>> +    if (!avio_ctx_buffer ) {
>> +        ret = AVERROR(ENOMEM);
>> +        avformat_free_context(pls->ctx);
>> +        pls->ctx = NULL;
>> +        goto fail;
>> +    }
>> +    ffio_init_context(&pls->pb, avio_ctx_buffer , INITIAL_BUFFER_SIZE, 0, pls, read_data, NULL, NULL);
>> +    pls->pb.seekable = 0;
>> +
>> +    if ((ret = ff_copy_whiteblacklists(pls->ctx, s)) < 0)
>> +        goto fail;
>> +
>> +    pls->ctx->flags = AVFMT_FLAG_CUSTOM_IO;
>> +    ret = av_probe_input_buffer(&pls->pb, &in_fmt, "", NULL, 0, 0);
>> +    if (ret < 0) {
>> +        av_log(s, AV_LOG_ERROR, "Error when loading segment, playlist %d\n", pls->cur_seq_no);
>> +        avformat_free_context(pls->ctx);
>> +        pls->ctx = NULL;
>> +        goto fail;
>> +    }
>> +
>> +    pls->ctx->pb = &pls->pb;
>> +    pls->ctx->io_open  = nested_io_open;
>> +
>> +    ret = avformat_open_input(&pls->ctx, "", in_fmt, &in_fmt_opts);
>> +    av_dict_free(&in_fmt_opts);
>> +    if (ret < 0)
>> +        goto fail;
>> +    ret = avformat_find_stream_info(pls->ctx, NULL);
>> +    if (ret < 0)
>> +        goto fail;
>> +
>> +fail:
>> +    return ret;
>> +}
>> +
>>  static int hls_read_packet(AVFormatContext *s, AVPacket *pkt)
>>  {
>>      HLSContext *c = s->priv_data;
>> @@ -2351,6 +2413,11 @@ static int hls_read_seek(AVFormatContext *s, int stream_index,
>>              pls->seek_stream_index = -1;
>>              pls->seek_flags |= AVSEEK_FLAG_ANY;
>>          }
>> +
>> +        if (pls->ctx && pls->ctx->iformat && strcmp(pls->ctx->iformat->name, "mov,mp4,m4a,3gp,3g2,mj2") == 0) {
>> +            pls->init_sec_buf_read_offset = 0;
>> +            reopen_demux_for_component(s, pls);
>> +        }
>>      }
>>        c->cur_timestamp = seek_timestamp;
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe”.

I have previously submitted another means of achieving this here:

https://patchwork.ffmpeg.org/project/ffmpeg/list/?series=1018 <https://patchwork.ffmpeg.org/project/ffmpeg/list/?series=1018>

My approach was to mirror in mp4 demux the same support that mpeg2ts has for detecting a position reset on the stream. And to improve the seek semantics in HLS demux.

Some older history here:

https://trac.ffmpeg.org/ticket/7485 <https://trac.ffmpeg.org/ticket/7485>