[FFmpeg-devel,v2,2/2] avformat/hlsenc: write temp file for append single file by encryption mode

fix ticket: 8783
Because in single file by encryption mode, it cannot get the last one
block of the file, it need ff_format_io_close for get full file size,
then hlsenc can get the total size of the encryption content,
so write the content into temp file first, and get the temp file content
append the temp file content into append to single file, then hlsenc can
get the correct file/content size and offset.

Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
---
 libavformat/hlsenc.c | 70 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 68 insertions(+), 2 deletions(-)

Steven Liu <lq@chinaffmpeg.org> 于2020年7月22日周三 下午5:16写道：
>
> fix ticket: 8783
> Because in single file by encryption mode, it cannot get the last one
> block of the file, it need ff_format_io_close for get full file size,
> then hlsenc can get the total size of the encryption content,
> so write the content into temp file first, and get the temp file content
> append the temp file content into append to single file, then hlsenc can
> get the correct file/content size and offset.
>
> Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
> ---
>  libavformat/hlsenc.c | 70 ++++++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 68 insertions(+), 2 deletions(-)
>
> diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
> index df84e6487d..f1e4a302d8 100644
> --- a/libavformat/hlsenc.c
> +++ b/libavformat/hlsenc.c
> @@ -69,6 +69,7 @@ typedef enum {
>  #define KEYSIZE 16
>  #define LINE_BUFFER_SIZE MAX_URL_SIZE
>  #define HLS_MICROSECOND_UNIT   1000000
> +#define BUFSIZE (16 * 1024)
>  #define POSTFIX_PATTERN "_%d"
>
>  typedef struct HLSSegment {
> @@ -119,6 +120,7 @@ typedef struct VariantStream {
>      ff_const59 AVOutputFormat *oformat;
>      ff_const59 AVOutputFormat *vtt_oformat;
>      AVIOContext *out;
> +    AVIOContext *out_single_file;
>      int packets_written;
>      int init_range_length;
>      uint8_t *temp_buffer;
> @@ -149,6 +151,7 @@ typedef struct VariantStream {
>      HLSSegment *last_segment;
>      HLSSegment *old_segments;
>
> +    char *basename_tmp;
>      char *basename;
>      char *vtt_basename;
>      char *vtt_m3u8_name;
> @@ -1722,12 +1725,34 @@ static int hls_start(AVFormatContext *s, VariantStream *vs)
>              av_opt_set(oc->priv_data, "mpegts_flags", "resend_headers", 0);
>          }
>          if (c->flags & HLS_SINGLE_FILE) {
> +            if (c->key_info_file || c->encrypt) {
> +                av_dict_set(&options, "encryption_key", vs->key_string, 0);
> +                av_dict_set(&options, "encryption_iv", vs->iv_string, 0);
> +
> +                /* Write temp file with cryption content */
> +                av_freep(&vs->basename_tmp);
> +                vs->basename_tmp = av_asprintf("crypto:%s.tmp", oc->url);
> +
> +                /* append temp file content into single file */
> +                av_freep(&vs->basename);
> +                vs->basename = av_asprintf("%s", oc->url);
> +            } else {
> +                vs->basename_tmp = vs->basename;
> +            }
>              set_http_options(s, &options, c);
> -            if ((err = hlsenc_io_open(s, &vs->out, oc->url, &options)) < 0) {
> +            if (!vs->out_single_file)
> +                if ((err = hlsenc_io_open(s, &vs->out_single_file, vs->basename, &options)) < 0) {
> +                    if (c->ignore_io_errors)
> +                        err = 0;
> +                    goto fail;
> +                }
> +
> +            if ((err = hlsenc_io_open(s, &vs->out, vs->basename_tmp, &options)) < 0) {
>                  if (c->ignore_io_errors)
>                      err = 0;
>                  goto fail;
>              }
> +
>          }
>      }
>      if (vs->vtt_basename) {
> @@ -2258,6 +2283,38 @@ static int hls_init_file_resend(AVFormatContext *s, VariantStream *vs)
>      return ret;
>  }
>
> +static int64_t append_single_file(AVFormatContext *s, VariantStream *vs)
> +{
> +    int ret = 0;
> +    int64_t read_byte = 0;
> +    int64_t total_size = 0;
> +    char *filename = NULL;
> +    char buf[BUFSIZE];
> +    AVFormatContext *oc = vs->avf;
> +
> +    hlsenc_io_close(s, &vs->out, vs->basename_tmp);
> +    filename = av_asprintf("%s.tmp", oc->url);
> +    ret = s->io_open(s, &vs->out, filename, AVIO_FLAG_READ, NULL);
> +    if (ret < 0) {
> +        av_free(filename);
> +        return ret;
> +    }
> +
> +    do {
> +        memset(buf, 0, sizeof(BUFSIZE));
> +        read_byte = avio_read(vs->out, buf, BUFSIZE);
> +        avio_write(vs->out_single_file, buf, read_byte);
> +        if (read_byte > 0) {
> +            total_size += read_byte;
> +            ret = total_size;
> +        }
> +    } while (read_byte > 0);
> +
> +    hlsenc_io_close(s, &vs->out, filename);
> +    av_free(filename);
> +
> +    return ret;
> +}
>  static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
>  {
>      HLSContext *hls = s->priv_data;
> @@ -2383,6 +2440,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
>                  return ret;
>              }
>              vs->size = range_length;
> +            if (hls->key_info_file || hls->encrypt)
> +                vs->size = append_single_file(s, vs);
>          } else {
>              if (oc->url[0]) {
>                  proto = avio_find_protocol_name(oc->url);
> @@ -2484,6 +2543,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
>
>          if (hls->flags & HLS_SINGLE_FILE) {
>              vs->start_pos += vs->size;
> +            if (hls->key_info_file || hls->encrypt)
> +                ret = hls_start(s, vs);
>          } else if (hls->max_seg_size > 0) {
>              if (vs->size + vs->start_pos >= hls->max_seg_size) {
>                  vs->sequence++;
> @@ -2644,7 +2705,12 @@ static int hls_write_trailer(struct AVFormatContext *s)
>              if (ret < 0)
>                  av_log(s, AV_LOG_WARNING, "Failed to upload file '%s' at the end.\n", oc->url);
>          }
> -
> +        if (hls->flags & HLS_SINGLE_FILE) {
> +            if (hls->key_info_file || hls->encrypt) {
> +                vs->size = append_single_file(s, vs);
> +            }
> +            hlsenc_io_close(s, &vs->out_single_file, vs->basename);
> +        }
>  failed:
>          av_freep(&vs->temp_buffer);
>          av_dict_free(&options);
> --
> 2.25.0
>
>

ping for review :D

Thanks

Steven

Steven Liu <lingjiujianke@gmail.com> 于2020年7月25日周六 上午10:49写道：
>
> Steven Liu <lq@chinaffmpeg.org> 于2020年7月22日周三 下午5:16写道：
> >
> > fix ticket: 8783
> > Because in single file by encryption mode, it cannot get the last one
> > block of the file, it need ff_format_io_close for get full file size,
> > then hlsenc can get the total size of the encryption content,
> > so write the content into temp file first, and get the temp file content
> > append the temp file content into append to single file, then hlsenc can
> > get the correct file/content size and offset.
> >
> > Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
> > ---
> >  libavformat/hlsenc.c | 70 ++++++++++++++++++++++++++++++++++++++++++--
> >  1 file changed, 68 insertions(+), 2 deletions(-)
> >
> > diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
> > index df84e6487d..f1e4a302d8 100644
> > --- a/libavformat/hlsenc.c
> > +++ b/libavformat/hlsenc.c
> > @@ -69,6 +69,7 @@ typedef enum {
> >  #define KEYSIZE 16
> >  #define LINE_BUFFER_SIZE MAX_URL_SIZE
> >  #define HLS_MICROSECOND_UNIT   1000000
> > +#define BUFSIZE (16 * 1024)
> >  #define POSTFIX_PATTERN "_%d"
> >
> >  typedef struct HLSSegment {
> > @@ -119,6 +120,7 @@ typedef struct VariantStream {
> >      ff_const59 AVOutputFormat *oformat;
> >      ff_const59 AVOutputFormat *vtt_oformat;
> >      AVIOContext *out;
> > +    AVIOContext *out_single_file;
> >      int packets_written;
> >      int init_range_length;
> >      uint8_t *temp_buffer;
> > @@ -149,6 +151,7 @@ typedef struct VariantStream {
> >      HLSSegment *last_segment;
> >      HLSSegment *old_segments;
> >
> > +    char *basename_tmp;
> >      char *basename;
> >      char *vtt_basename;
> >      char *vtt_m3u8_name;
> > @@ -1722,12 +1725,34 @@ static int hls_start(AVFormatContext *s, VariantStream *vs)
> >              av_opt_set(oc->priv_data, "mpegts_flags", "resend_headers", 0);
> >          }
> >          if (c->flags & HLS_SINGLE_FILE) {
> > +            if (c->key_info_file || c->encrypt) {
> > +                av_dict_set(&options, "encryption_key", vs->key_string, 0);
> > +                av_dict_set(&options, "encryption_iv", vs->iv_string, 0);
> > +
> > +                /* Write temp file with cryption content */
> > +                av_freep(&vs->basename_tmp);
> > +                vs->basename_tmp = av_asprintf("crypto:%s.tmp", oc->url);
> > +
> > +                /* append temp file content into single file */
> > +                av_freep(&vs->basename);
> > +                vs->basename = av_asprintf("%s", oc->url);
> > +            } else {
> > +                vs->basename_tmp = vs->basename;
> > +            }
> >              set_http_options(s, &options, c);
> > -            if ((err = hlsenc_io_open(s, &vs->out, oc->url, &options)) < 0) {
> > +            if (!vs->out_single_file)
> > +                if ((err = hlsenc_io_open(s, &vs->out_single_file, vs->basename, &options)) < 0) {
> > +                    if (c->ignore_io_errors)
> > +                        err = 0;
> > +                    goto fail;
> > +                }
> > +
> > +            if ((err = hlsenc_io_open(s, &vs->out, vs->basename_tmp, &options)) < 0) {
> >                  if (c->ignore_io_errors)
> >                      err = 0;
> >                  goto fail;
> >              }
> > +
> >          }
> >      }
> >      if (vs->vtt_basename) {
> > @@ -2258,6 +2283,38 @@ static int hls_init_file_resend(AVFormatContext *s, VariantStream *vs)
> >      return ret;
> >  }
> >
> > +static int64_t append_single_file(AVFormatContext *s, VariantStream *vs)
> > +{
> > +    int ret = 0;
> > +    int64_t read_byte = 0;
> > +    int64_t total_size = 0;
> > +    char *filename = NULL;
> > +    char buf[BUFSIZE];
> > +    AVFormatContext *oc = vs->avf;
> > +
> > +    hlsenc_io_close(s, &vs->out, vs->basename_tmp);
> > +    filename = av_asprintf("%s.tmp", oc->url);
> > +    ret = s->io_open(s, &vs->out, filename, AVIO_FLAG_READ, NULL);
> > +    if (ret < 0) {
> > +        av_free(filename);
> > +        return ret;
> > +    }
> > +
> > +    do {
> > +        memset(buf, 0, sizeof(BUFSIZE));
> > +        read_byte = avio_read(vs->out, buf, BUFSIZE);
> > +        avio_write(vs->out_single_file, buf, read_byte);
> > +        if (read_byte > 0) {
> > +            total_size += read_byte;
> > +            ret = total_size;
> > +        }
> > +    } while (read_byte > 0);
> > +
> > +    hlsenc_io_close(s, &vs->out, filename);
> > +    av_free(filename);
> > +
> > +    return ret;
> > +}
> >  static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> >  {
> >      HLSContext *hls = s->priv_data;
> > @@ -2383,6 +2440,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> >                  return ret;
> >              }
> >              vs->size = range_length;
> > +            if (hls->key_info_file || hls->encrypt)
> > +                vs->size = append_single_file(s, vs);
> >          } else {
> >              if (oc->url[0]) {
> >                  proto = avio_find_protocol_name(oc->url);
> > @@ -2484,6 +2543,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> >
> >          if (hls->flags & HLS_SINGLE_FILE) {
> >              vs->start_pos += vs->size;
> > +            if (hls->key_info_file || hls->encrypt)
> > +                ret = hls_start(s, vs);
> >          } else if (hls->max_seg_size > 0) {
> >              if (vs->size + vs->start_pos >= hls->max_seg_size) {
> >                  vs->sequence++;
> > @@ -2644,7 +2705,12 @@ static int hls_write_trailer(struct AVFormatContext *s)
> >              if (ret < 0)
> >                  av_log(s, AV_LOG_WARNING, "Failed to upload file '%s' at the end.\n", oc->url);
> >          }
> > -
> > +        if (hls->flags & HLS_SINGLE_FILE) {
> > +            if (hls->key_info_file || hls->encrypt) {
> > +                vs->size = append_single_file(s, vs);
> > +            }
> > +            hlsenc_io_close(s, &vs->out_single_file, vs->basename);
> > +        }
> >  failed:
> >          av_freep(&vs->temp_buffer);
> >          av_dict_free(&options);
> > --
> > 2.25.0
> >
> >
>
> ping for review :D
Any suggestions?


Thanks
Steven

Steven Liu <lingjiujianke@gmail.com> 于2020年8月2日周日 下午10:35写道：
>
> Steven Liu <lingjiujianke@gmail.com> 于2020年7月25日周六 上午10:49写道：
> >
> > Steven Liu <lq@chinaffmpeg.org> 于2020年7月22日周三 下午5:16写道：
> > >
> > > fix ticket: 8783
> > > Because in single file by encryption mode, it cannot get the last one
> > > block of the file, it need ff_format_io_close for get full file size,
> > > then hlsenc can get the total size of the encryption content,
> > > so write the content into temp file first, and get the temp file content
> > > append the temp file content into append to single file, then hlsenc can
> > > get the correct file/content size and offset.
> > >
> > > Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
> > > ---
> > >  libavformat/hlsenc.c | 70 ++++++++++++++++++++++++++++++++++++++++++--
> > >  1 file changed, 68 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c
> > > index df84e6487d..f1e4a302d8 100644
> > > --- a/libavformat/hlsenc.c
> > > +++ b/libavformat/hlsenc.c
> > > @@ -69,6 +69,7 @@ typedef enum {
> > >  #define KEYSIZE 16
> > >  #define LINE_BUFFER_SIZE MAX_URL_SIZE
> > >  #define HLS_MICROSECOND_UNIT   1000000
> > > +#define BUFSIZE (16 * 1024)
> > >  #define POSTFIX_PATTERN "_%d"
> > >
> > >  typedef struct HLSSegment {
> > > @@ -119,6 +120,7 @@ typedef struct VariantStream {
> > >      ff_const59 AVOutputFormat *oformat;
> > >      ff_const59 AVOutputFormat *vtt_oformat;
> > >      AVIOContext *out;
> > > +    AVIOContext *out_single_file;
> > >      int packets_written;
> > >      int init_range_length;
> > >      uint8_t *temp_buffer;
> > > @@ -149,6 +151,7 @@ typedef struct VariantStream {
> > >      HLSSegment *last_segment;
> > >      HLSSegment *old_segments;
> > >
> > > +    char *basename_tmp;
> > >      char *basename;
> > >      char *vtt_basename;
> > >      char *vtt_m3u8_name;
> > > @@ -1722,12 +1725,34 @@ static int hls_start(AVFormatContext *s, VariantStream *vs)
> > >              av_opt_set(oc->priv_data, "mpegts_flags", "resend_headers", 0);
> > >          }
> > >          if (c->flags & HLS_SINGLE_FILE) {
> > > +            if (c->key_info_file || c->encrypt) {
> > > +                av_dict_set(&options, "encryption_key", vs->key_string, 0);
> > > +                av_dict_set(&options, "encryption_iv", vs->iv_string, 0);
> > > +
> > > +                /* Write temp file with cryption content */
> > > +                av_freep(&vs->basename_tmp);
> > > +                vs->basename_tmp = av_asprintf("crypto:%s.tmp", oc->url);
> > > +
> > > +                /* append temp file content into single file */
> > > +                av_freep(&vs->basename);
> > > +                vs->basename = av_asprintf("%s", oc->url);
> > > +            } else {
> > > +                vs->basename_tmp = vs->basename;
> > > +            }
> > >              set_http_options(s, &options, c);
> > > -            if ((err = hlsenc_io_open(s, &vs->out, oc->url, &options)) < 0) {
> > > +            if (!vs->out_single_file)
> > > +                if ((err = hlsenc_io_open(s, &vs->out_single_file, vs->basename, &options)) < 0) {
> > > +                    if (c->ignore_io_errors)
> > > +                        err = 0;
> > > +                    goto fail;
> > > +                }
> > > +
> > > +            if ((err = hlsenc_io_open(s, &vs->out, vs->basename_tmp, &options)) < 0) {
> > >                  if (c->ignore_io_errors)
> > >                      err = 0;
> > >                  goto fail;
> > >              }
> > > +
> > >          }
> > >      }
> > >      if (vs->vtt_basename) {
> > > @@ -2258,6 +2283,38 @@ static int hls_init_file_resend(AVFormatContext *s, VariantStream *vs)
> > >      return ret;
> > >  }
> > >
> > > +static int64_t append_single_file(AVFormatContext *s, VariantStream *vs)
> > > +{
> > > +    int ret = 0;
> > > +    int64_t read_byte = 0;
> > > +    int64_t total_size = 0;
> > > +    char *filename = NULL;
> > > +    char buf[BUFSIZE];
> > > +    AVFormatContext *oc = vs->avf;
> > > +
> > > +    hlsenc_io_close(s, &vs->out, vs->basename_tmp);
> > > +    filename = av_asprintf("%s.tmp", oc->url);
> > > +    ret = s->io_open(s, &vs->out, filename, AVIO_FLAG_READ, NULL);
> > > +    if (ret < 0) {
> > > +        av_free(filename);
> > > +        return ret;
> > > +    }
> > > +
> > > +    do {
> > > +        memset(buf, 0, sizeof(BUFSIZE));
> > > +        read_byte = avio_read(vs->out, buf, BUFSIZE);
> > > +        avio_write(vs->out_single_file, buf, read_byte);
> > > +        if (read_byte > 0) {
> > > +            total_size += read_byte;
> > > +            ret = total_size;
> > > +        }
> > > +    } while (read_byte > 0);
> > > +
> > > +    hlsenc_io_close(s, &vs->out, filename);
> > > +    av_free(filename);
> > > +
> > > +    return ret;
> > > +}
> > >  static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> > >  {
> > >      HLSContext *hls = s->priv_data;
> > > @@ -2383,6 +2440,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> > >                  return ret;
> > >              }
> > >              vs->size = range_length;
> > > +            if (hls->key_info_file || hls->encrypt)
> > > +                vs->size = append_single_file(s, vs);
> > >          } else {
> > >              if (oc->url[0]) {
> > >                  proto = avio_find_protocol_name(oc->url);
> > > @@ -2484,6 +2543,8 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt)
> > >
> > >          if (hls->flags & HLS_SINGLE_FILE) {
> > >              vs->start_pos += vs->size;
> > > +            if (hls->key_info_file || hls->encrypt)
> > > +                ret = hls_start(s, vs);
> > >          } else if (hls->max_seg_size > 0) {
> > >              if (vs->size + vs->start_pos >= hls->max_seg_size) {
> > >                  vs->sequence++;
> > > @@ -2644,7 +2705,12 @@ static int hls_write_trailer(struct AVFormatContext *s)
> > >              if (ret < 0)
> > >                  av_log(s, AV_LOG_WARNING, "Failed to upload file '%s' at the end.\n", oc->url);
> > >          }
> > > -
> > > +        if (hls->flags & HLS_SINGLE_FILE) {
> > > +            if (hls->key_info_file || hls->encrypt) {
> > > +                vs->size = append_single_file(s, vs);
> > > +            }
> > > +            hlsenc_io_close(s, &vs->out_single_file, vs->basename);
> > > +        }
> > >  failed:
> > >          av_freep(&vs->temp_buffer);
> > >          av_dict_free(&options);
> > > --
> > > 2.25.0
> > >
> > >
> >
> > ping for review :D
> Any suggestions?
Pushed

>
>
> Thanks
> Steven