diff mbox

[FFmpeg-devel,1/2] Add experimental muxing support for FLAC in ISO BMFF (MP4).

Message ID 20161101232152.GD8366@brak.lan
State Superseded
Headers show

Commit Message

Matthew Gregan Nov. 1, 2016, 11:21 p.m. UTC 
At 2016-11-01T00:56:45-0300, James Almer wrote:
> The muxer gets the extradata as created by the encoder only during init().
> Even if the encoder changes it in any way afterwards, it will never make
> it to the muxer. That's why packet side data is used for this.
> 
> The reason the encoder used the AVCodecContext extradata to store the
> updated Streaminfo before creating the packet side data is most likely
> because it was an already allocated buffer. Technically, it's not the
> "proper" thing to do, but it also doesn't seem to make a difference.

Makes sense, thanks for the explanation.

Updated patch attached with the following changes:
- Uses libavcodec/flac.h for the FLAC_STREAMINFO_SIZE and
  FLAC_METADATA_TYPE_STREAMINFO definitions, per the feedback on the demuxer
  patch.
- Fixes the sample size in bits written to the 'fLaC' audio box.
  Thanks to Vittorio Giovara for pointing this out.

Comments

Matthew Gregan Nov. 15, 2016, 1:38 a.m. UTC | #1 
Ping? It's been a couple of weeks since the last comment and patch update.
Can I help move this forward somehow?

Firefox will be shipping experimental support for playback of FLAC in MP4
based on the current spec in Firefox 51.
James Almer Nov. 15, 2016, 2:02 a.m. UTC | #2 
On 11/14/2016 10:38 PM, Matthew Gregan wrote:
> Ping? It's been a couple of weeks since the last comment and patch update.
> Can I help move this forward somehow?
> 
> Firefox will be shipping experimental support for playback of FLAC in MP4
> based on the current spec in Firefox 51.

CCing Matthieu Bouron since he's listed as movenc maintainer. Otherwise
I'll push it later this week as they seem ok and the code is under a
strict experimental check anyway.
James Almer Nov. 18, 2016, 1:57 a.m. UTC | #3 
On 11/1/2016 8:21 PM, Matthew Gregan wrote:
> At 2016-11-01T00:56:45-0300, James Almer wrote:
>> > The muxer gets the extradata as created by the encoder only during init().
>> > Even if the encoder changes it in any way afterwards, it will never make
>> > it to the muxer. That's why packet side data is used for this.
>> > 
>> > The reason the encoder used the AVCodecContext extradata to store the
>> > updated Streaminfo before creating the packet side data is most likely
>> > because it was an already allocated buffer. Technically, it's not the
>> > "proper" thing to do, but it also doesn't seem to make a difference.
> Makes sense, thanks for the explanation.
> 
> Updated patch attached with the following changes:
> - Uses libavcodec/flac.h for the FLAC_STREAMINFO_SIZE and
>   FLAC_METADATA_TYPE_STREAMINFO definitions, per the feedback on the demuxer
>   patch.
> - Fixes the sample size in bits written to the 'fLaC' audio box.
>   Thanks to Vittorio Giovara for pointing this out.
> 
> 
> 0001-Add-experimental-muxing-support-for-FLAC-in-ISO-BMFF.patch
> 
> 
> From e08e1b5041a331d640d190c39c00bdc61e81243c Mon Sep 17 00:00:00 2001
> From: Matthew Gregan <kinetik@flim.org>
> Date: Thu, 20 Oct 2016 17:28:11 +1300
> Subject: [PATCH 1/2] Add experimental muxing support for FLAC in ISO BMFF
>  (MP4).
> 
> Based on the draft spec at https://git.xiph.org/?p=flac.git;a=blob;f=doc/isoflac.txt
> 
> '-strict -2' is required to create files in this format.
> 
> Signed-off-by: Matthew Gregan <kinetik@flim.org>
> ---
>  libavformat/isom.c   |  2 ++
>  libavformat/movenc.c | 46 +++++++++++++++++++++++++++++++++++++++++++---
>  2 files changed, 45 insertions(+), 3 deletions(-)
> 
> diff --git a/libavformat/isom.c b/libavformat/isom.c
> index ab79e22..aacbe43 100644
> --- a/libavformat/isom.c
> +++ b/libavformat/isom.c
> @@ -60,6 +60,7 @@ const AVCodecTag ff_mp4_obj_type[] = {
>      { AV_CODEC_ID_EAC3        , 0xA6 },
>      { AV_CODEC_ID_DTS         , 0xA9 }, /* mp4ra.org */
>      { AV_CODEC_ID_VP9         , 0xC0 }, /* nonstandard, update when there is a standard value */
> +    { AV_CODEC_ID_FLAC        , 0xC1 }, /* nonstandard, update when there is a standard value */
>      { AV_CODEC_ID_TSCC2       , 0xD0 }, /* nonstandard, camtasia uses it */
>      { AV_CODEC_ID_VORBIS      , 0xDD }, /* nonstandard, gpac uses it */
>      { AV_CODEC_ID_DVD_SUBTITLE, 0xE0 }, /* nonstandard, see unsupported-embedded-subs-2.mp4 */
> @@ -345,6 +346,7 @@ const AVCodecTag ff_codec_movaudio_tags[] = {
>      { AV_CODEC_ID_WMAV2,           MKTAG('W', 'M', 'A', '2') },
>      { AV_CODEC_ID_EVRC,            MKTAG('s', 'e', 'v', 'c') }, /* 3GPP2 */
>      { AV_CODEC_ID_SMV,             MKTAG('s', 's', 'm', 'v') }, /* 3GPP2 */
> +    { AV_CODEC_ID_FLAC,            MKTAG('f', 'L', 'a', 'C') }, /* nonstandard */

This is enables muxing into mov, which bypasses the experimental check below
since mov isn't MODE_MP4.

>      { AV_CODEC_ID_NONE, 0 },
>  };
>  
> diff --git a/libavformat/movenc.c b/libavformat/movenc.c
> index 6228192..06f6dc6 100644
> --- a/libavformat/movenc.c
> +++ b/libavformat/movenc.c
> @@ -33,6 +33,7 @@
>  #include "avc.h"
>  #include "libavcodec/ac3_parser.h"
>  #include "libavcodec/dnxhddata.h"
> +#include "libavcodec/flac.h"
>  #include "libavcodec/get_bits.h"
>  #include "libavcodec/put_bits.h"
>  #include "libavcodec/vc1_common.h"
> @@ -654,6 +655,26 @@ static int mov_write_wfex_tag(AVFormatContext *s, AVIOContext *pb, MOVTrack *tra
>      return update_size(pb, pos);
>  }
>  
> +static int mov_write_dfla_tag(AVIOContext *pb, MOVTrack *track)
> +{
> +    int64_t pos = avio_tell(pb);
> +    avio_wb32(pb, 0);
> +    ffio_wfourcc(pb, "dfLa");
> +    avio_w8(pb, 0); /* version */
> +    avio_wb24(pb, 0); /* flags */
> +
> +    /* Expect the encoder to pass a METADATA_BLOCK_TYPE_STREAMINFO. */
> +    if (track->par->extradata_size != FLAC_STREAMINFO_SIZE)
> +      return AVERROR_INVALIDDATA;
> +
> +    /* TODO: Write other METADATA_BLOCK_TYPEs if the encoder makes them available. */
> +    avio_w8(pb, 1 << 7 | FLAC_METADATA_TYPE_STREAMINFO); /* LastMetadataBlockFlag << 7 | BlockType */
> +    avio_wb24(pb, track->par->extradata_size); /* Length */
> +    avio_write(pb, track->par->extradata, track->par->extradata_size); /* BlockData[Length] */
> +
> +    return update_size(pb, pos);
> +}
> +
>  static int mov_write_chan_tag(AVFormatContext *s, AVIOContext *pb, MOVTrack *track)
>  {
>      uint32_t layout_tag, bitmap;
> @@ -963,8 +984,13 @@ static int mov_write_audio_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
>                  avio_wb16(pb, 16);
>              avio_wb16(pb, track->audio_vbr ? -2 : 0); /* compression ID */
>          } else { /* reserved for mp4/3gp */
> -            avio_wb16(pb, 2);
> -            avio_wb16(pb, 16);
> +            if (track->par->codec_id == AV_CODEC_ID_FLAC) {
> +                avio_wb16(pb, track->par->channels);
> +                avio_wb16(pb, track->par->bits_per_raw_sample);
> +            } else {
> +                avio_wb16(pb, 2);
> +                avio_wb16(pb, 16);
> +            }
>              avio_wb16(pb, 0);
>          }
>  
> @@ -1009,6 +1035,8 @@ static int mov_write_audio_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
>          mov_write_extradata_tag(pb, track);
>      else if (track->par->codec_id == AV_CODEC_ID_WMAPRO)
>          mov_write_wfex_tag(s, pb, track);
> +    else if (track->par->codec_id == AV_CODEC_ID_FLAC)
> +        mov_write_dfla_tag(pb, track);
>      else if (track->vos_len > 0)
>          mov_write_glbl_tag(pb, track);
>  
> @@ -1177,6 +1205,7 @@ static int mp4_get_codec_tag(AVFormatContext *s, MOVTrack *track)
>      else if (track->par->codec_id == AV_CODEC_ID_DIRAC)     tag = MKTAG('d','r','a','c');
>      else if (track->par->codec_id == AV_CODEC_ID_MOV_TEXT)  tag = MKTAG('t','x','3','g');
>      else if (track->par->codec_id == AV_CODEC_ID_VC1)       tag = MKTAG('v','c','-','1');
> +    else if (track->par->codec_id == AV_CODEC_ID_FLAC)      tag = MKTAG('f','L','a','C');
>      else if (track->par->codec_type == AVMEDIA_TYPE_VIDEO)  tag = MKTAG('m','p','4','v');
>      else if (track->par->codec_type == AVMEDIA_TYPE_AUDIO)  tag = MKTAG('m','p','4','a');
>      else if (track->par->codec_id == AV_CODEC_ID_DVD_SUBTITLE)  tag = MKTAG('m','p','4','s');
> @@ -5019,7 +5048,8 @@ static int mov_write_single_packet(AVFormatContext *s, AVPacket *pkt)
>                      trk->start_cts = 0;
>              }
>  
> -            if (trk->par->codec_id == AV_CODEC_ID_MP4ALS) {
> +            if (trk->par->codec_id == AV_CODEC_ID_MP4ALS ||
> +                trk->par->codec_id == AV_CODEC_ID_FLAC) {
>                  int side_size = 0;
>                  uint8_t *side = av_packet_get_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, &side_size);
>                  if (side && side_size > 0 && (side_size != par->extradata_size || memcmp(side, par->extradata, side_size))) {
> @@ -5733,6 +5763,16 @@ static int mov_init(AVFormatContext *s)
>                             i, track->par->sample_rate);
>                  }
>              }
> +            if (track->mode == MODE_MP4 &&
> +                track->par->codec_id == AV_CODEC_ID_FLAC) {
> +                if (s->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
> +                    av_log(s, AV_LOG_ERROR,
> +                           "FLAC in MP4 support is experimental, add "
> +                           "'-strict %d' if you want to use it.\n",
> +                           FF_COMPLIANCE_EXPERIMENTAL);
> +                    return AVERROR_EXPERIMENTAL;
> +                }
> +            }
>          } else if (st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
>              track->timescale = st->time_base.den;
>          } else if (st->codecpar->codec_type == AVMEDIA_TYPE_DATA) {
> -- 2.10.1
> 
> 
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
diff mbox

Patch

From e08e1b5041a331d640d190c39c00bdc61e81243c Mon Sep 17 00:00:00 2001
From: Matthew Gregan <kinetik@flim.org>
Date: Thu, 20 Oct 2016 17:28:11 +1300
Subject: [PATCH 1/2] Add experimental muxing support for FLAC in ISO BMFF
 (MP4).

Based on the draft spec at https://git.xiph.org/?p=flac.git;a=blob;f=doc/isoflac.txt

'-strict -2' is required to create files in this format.

Signed-off-by: Matthew Gregan <kinetik@flim.org>
---
 libavformat/isom.c   |  2 ++
 libavformat/movenc.c | 46 +++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/libavformat/isom.c b/libavformat/isom.c
index ab79e22..aacbe43 100644
--- a/libavformat/isom.c
+++ b/libavformat/isom.c
@@ -60,6 +60,7 @@  const AVCodecTag ff_mp4_obj_type[] = {
     { AV_CODEC_ID_EAC3        , 0xA6 },
     { AV_CODEC_ID_DTS         , 0xA9 }, /* mp4ra.org */
     { AV_CODEC_ID_VP9         , 0xC0 }, /* nonstandard, update when there is a standard value */
+    { AV_CODEC_ID_FLAC        , 0xC1 }, /* nonstandard, update when there is a standard value */
     { AV_CODEC_ID_TSCC2       , 0xD0 }, /* nonstandard, camtasia uses it */
     { AV_CODEC_ID_VORBIS      , 0xDD }, /* nonstandard, gpac uses it */
     { AV_CODEC_ID_DVD_SUBTITLE, 0xE0 }, /* nonstandard, see unsupported-embedded-subs-2.mp4 */
@@ -345,6 +346,7 @@  const AVCodecTag ff_codec_movaudio_tags[] = {
     { AV_CODEC_ID_WMAV2,           MKTAG('W', 'M', 'A', '2') },
     { AV_CODEC_ID_EVRC,            MKTAG('s', 'e', 'v', 'c') }, /* 3GPP2 */
     { AV_CODEC_ID_SMV,             MKTAG('s', 's', 'm', 'v') }, /* 3GPP2 */
+    { AV_CODEC_ID_FLAC,            MKTAG('f', 'L', 'a', 'C') }, /* nonstandard */
     { AV_CODEC_ID_NONE, 0 },
 };
 
diff --git a/libavformat/movenc.c b/libavformat/movenc.c
index 6228192..06f6dc6 100644
--- a/libavformat/movenc.c
+++ b/libavformat/movenc.c
@@ -33,6 +33,7 @@ 
 #include "avc.h"
 #include "libavcodec/ac3_parser.h"
 #include "libavcodec/dnxhddata.h"
+#include "libavcodec/flac.h"
 #include "libavcodec/get_bits.h"
 #include "libavcodec/put_bits.h"
 #include "libavcodec/vc1_common.h"
@@ -654,6 +655,26 @@  static int mov_write_wfex_tag(AVFormatContext *s, AVIOContext *pb, MOVTrack *tra
     return update_size(pb, pos);
 }
 
+static int mov_write_dfla_tag(AVIOContext *pb, MOVTrack *track)
+{
+    int64_t pos = avio_tell(pb);
+    avio_wb32(pb, 0);
+    ffio_wfourcc(pb, "dfLa");
+    avio_w8(pb, 0); /* version */
+    avio_wb24(pb, 0); /* flags */
+
+    /* Expect the encoder to pass a METADATA_BLOCK_TYPE_STREAMINFO. */
+    if (track->par->extradata_size != FLAC_STREAMINFO_SIZE)
+      return AVERROR_INVALIDDATA;
+
+    /* TODO: Write other METADATA_BLOCK_TYPEs if the encoder makes them available. */
+    avio_w8(pb, 1 << 7 | FLAC_METADATA_TYPE_STREAMINFO); /* LastMetadataBlockFlag << 7 | BlockType */
+    avio_wb24(pb, track->par->extradata_size); /* Length */
+    avio_write(pb, track->par->extradata, track->par->extradata_size); /* BlockData[Length] */
+
+    return update_size(pb, pos);
+}
+
 static int mov_write_chan_tag(AVFormatContext *s, AVIOContext *pb, MOVTrack *track)
 {
     uint32_t layout_tag, bitmap;
@@ -963,8 +984,13 @@  static int mov_write_audio_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
                 avio_wb16(pb, 16);
             avio_wb16(pb, track->audio_vbr ? -2 : 0); /* compression ID */
         } else { /* reserved for mp4/3gp */
-            avio_wb16(pb, 2);
-            avio_wb16(pb, 16);
+            if (track->par->codec_id == AV_CODEC_ID_FLAC) {
+                avio_wb16(pb, track->par->channels);
+                avio_wb16(pb, track->par->bits_per_raw_sample);
+            } else {
+                avio_wb16(pb, 2);
+                avio_wb16(pb, 16);
+            }
             avio_wb16(pb, 0);
         }
 
@@ -1009,6 +1035,8 @@  static int mov_write_audio_tag(AVFormatContext *s, AVIOContext *pb, MOVMuxContex
         mov_write_extradata_tag(pb, track);
     else if (track->par->codec_id == AV_CODEC_ID_WMAPRO)
         mov_write_wfex_tag(s, pb, track);
+    else if (track->par->codec_id == AV_CODEC_ID_FLAC)
+        mov_write_dfla_tag(pb, track);
     else if (track->vos_len > 0)
         mov_write_glbl_tag(pb, track);
 
@@ -1177,6 +1205,7 @@  static int mp4_get_codec_tag(AVFormatContext *s, MOVTrack *track)
     else if (track->par->codec_id == AV_CODEC_ID_DIRAC)     tag = MKTAG('d','r','a','c');
     else if (track->par->codec_id == AV_CODEC_ID_MOV_TEXT)  tag = MKTAG('t','x','3','g');
     else if (track->par->codec_id == AV_CODEC_ID_VC1)       tag = MKTAG('v','c','-','1');
+    else if (track->par->codec_id == AV_CODEC_ID_FLAC)      tag = MKTAG('f','L','a','C');
     else if (track->par->codec_type == AVMEDIA_TYPE_VIDEO)  tag = MKTAG('m','p','4','v');
     else if (track->par->codec_type == AVMEDIA_TYPE_AUDIO)  tag = MKTAG('m','p','4','a');
     else if (track->par->codec_id == AV_CODEC_ID_DVD_SUBTITLE)  tag = MKTAG('m','p','4','s');
@@ -5019,7 +5048,8 @@  static int mov_write_single_packet(AVFormatContext *s, AVPacket *pkt)
                     trk->start_cts = 0;
             }
 
-            if (trk->par->codec_id == AV_CODEC_ID_MP4ALS) {
+            if (trk->par->codec_id == AV_CODEC_ID_MP4ALS ||
+                trk->par->codec_id == AV_CODEC_ID_FLAC) {
                 int side_size = 0;
                 uint8_t *side = av_packet_get_side_data(pkt, AV_PKT_DATA_NEW_EXTRADATA, &side_size);
                 if (side && side_size > 0 && (side_size != par->extradata_size || memcmp(side, par->extradata, side_size))) {
@@ -5733,6 +5763,16 @@  static int mov_init(AVFormatContext *s)
                            i, track->par->sample_rate);
                 }
             }
+            if (track->mode == MODE_MP4 &&
+                track->par->codec_id == AV_CODEC_ID_FLAC) {
+                if (s->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
+                    av_log(s, AV_LOG_ERROR,
+                           "FLAC in MP4 support is experimental, add "
+                           "'-strict %d' if you want to use it.\n",
+                           FF_COMPLIANCE_EXPERIMENTAL);
+                    return AVERROR_EXPERIMENTAL;
+                }
+            }
         } else if (st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) {
             track->timescale = st->time_base.den;
         } else if (st->codecpar->codec_type == AVMEDIA_TYPE_DATA) {
-- 
2.10.1