diff mbox series

[FFmpeg-devel,2/2] avformat/flvdec: support enhanced flv PacketTypeMetadata

Message ID tencent_B9DCD1B361EE91197340C756B867C44C8A0A@qq.com
State New
Headers show
Series [FFmpeg-devel,1/2] avformat/flvenc: support enhanced flv PacketTypeMetadata | expand

Checks

Context Check Description
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_fate_x86 success Make fate finished
yinshiyou/make_loongarch64 warning New warnings during build
andriy/make_x86 warning New warnings during build

Commit Message

Diego Felix de Souza via ffmpeg-devel Nov. 15, 2023, 2:40 p.m. UTC 
From: Zhu Pengfei <411294962@qq.com>

Signed-off-by: Zhu Pengfei <411294962@qq.com>
---
 libavformat/flvdec.c | 171 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 170 insertions(+), 1 deletion(-)

Comments

James Almer Dec. 4, 2023, 9:32 p.m. UTC | #1 
On 11/15/2023 11:40 AM, zhupengfei via ffmpeg-devel wrote:
> From: Zhu Pengfei <411294962@qq.com>
> 
> Signed-off-by: Zhu Pengfei <411294962@qq.com>
> ---
>   libavformat/flvdec.c | 171 ++++++++++++++++++++++++++++++++++++++++++-
>   1 file changed, 170 insertions(+), 1 deletion(-)
> 
> diff --git a/libavformat/flvdec.c b/libavformat/flvdec.c
> index e25b5bd163..46bb0825ca 100644
> --- a/libavformat/flvdec.c
> +++ b/libavformat/flvdec.c
> @@ -34,6 +34,7 @@
>   #include "libavutil/intfloat.h"
>   #include "libavutil/intreadwrite.h"
>   #include "libavutil/mathematics.h"
> +#include "libavutil/mastering_display_metadata.h"
>   #include "avformat.h"
>   #include "demux.h"
>   #include "internal.h"
> @@ -45,6 +46,28 @@
>   
>   #define MAX_DEPTH 16      ///< arbitrary limit to prevent unbounded recursion
>   
> +typedef struct FLVMasteringMeta {
> +    double r_x;
> +    double r_y;
> +    double g_x;
> +    double g_y;
> +    double b_x;
> +    double b_y;
> +    double white_x;
> +    double white_y;
> +    double max_luminance;
> +    double min_luminance;
> +} FLVMasteringMeta;
> +
> +typedef struct FLVMetaVideoColor {
> +    uint64_t matrix_coefficients;
> +    uint64_t transfer_characteristics;
> +    uint64_t primaries;
> +    uint64_t max_cll;
> +    uint64_t max_fall;
> +    FLVMasteringMeta mastering_meta;
> +} FLVMetaVideoColor;
> +
>   typedef struct FLVContext {
>       const AVClass *class; ///< Class for private options.
>       int trust_metadata;   ///< configure streams according onMetaData
> @@ -80,6 +103,8 @@ typedef struct FLVContext {
>       int64_t time_offset;
>       int64_t time_pos;
>   
> +    FLVMetaVideoColor *metaVideoColor;
> +    int meta_color_info_flag;
>   } FLVContext;
>   
>   /* AMF date type */
> @@ -524,6 +549,7 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream,
>       FLVContext *flv = s->priv_data;
>       AVIOContext *ioc;
>       AMFDataType amf_type;
> +    FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
>       char str_val[1024];
>       double num_val;
>       amf_date date;
> @@ -655,6 +681,36 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream,
>                       } else if (!strcmp(key, "height") && vpar) {
>                           vpar->height = num_val;
>                       }
> +                } else if (!strcmp(key, "colorPrimaries") && meta_video_color) {

You should put this inside an "else if (meta_video_color)" block, 
instead of checking for meta_video_color with every single string.

See how it's done for flv->trust_metadata above.

> +                    meta_video_color->primaries = num_val;
> +                } else if (!strcmp(key, "transferCharacteristics") && meta_video_color) {
> +                    meta_video_color->transfer_characteristics = num_val;
> +                } else if (!strcmp(key, "matrixCoefficients") && meta_video_color) {
> +                    meta_video_color->matrix_coefficients = num_val;
> +                } else if (!strcmp(key, "maxFall") && meta_video_color) {
> +                    meta_video_color->max_fall = num_val;
> +                } else if (!strcmp(key, "maxCLL") && meta_video_color) {
> +                    meta_video_color->max_cll = num_val;
> +                } else if (!strcmp(key, "redX") && meta_video_color) {
> +                    meta_video_color->mastering_meta.r_x = num_val;
> +                } else if (!strcmp(key, "redY") && meta_video_color) {
> +                    meta_video_color->mastering_meta.r_y = num_val;
> +                } else if (!strcmp(key, "greenX") && meta_video_color) {
> +                    meta_video_color->mastering_meta.g_x = num_val;
> +                } else if (!strcmp(key, "greenY") && meta_video_color) {
> +                    meta_video_color->mastering_meta.g_y = num_val;
> +                } else if (!strcmp(key, "blueX") && meta_video_color) {
> +                    meta_video_color->mastering_meta.b_x = num_val;
> +                } else if (!strcmp(key, "blueY") && meta_video_color) {
> +                    meta_video_color->mastering_meta.b_y = num_val;
> +                } else if (!strcmp(key, "whitePointX") && meta_video_color) {
> +                    meta_video_color->mastering_meta.white_x = num_val;
> +                } else if (!strcmp(key, "whitePointY") && meta_video_color) {
> +                    meta_video_color->mastering_meta.white_y = num_val;
> +                } else if (!strcmp(key, "maxLuminance") && meta_video_color) {
> +                    meta_video_color->mastering_meta.max_luminance = num_val;
> +                } else if (!strcmp(key, "minLuminance") && meta_video_color) {
> +                    meta_video_color->mastering_meta.min_luminance = num_val;
>                   }
>               }
>               if (amf_type == AMF_DATA_TYPE_STRING) {
> @@ -824,6 +880,7 @@ static int flv_read_close(AVFormatContext *s)
>           av_freep(&flv->new_extradata[i]);
>       av_freep(&flv->keyframe_times);
>       av_freep(&flv->keyframe_filepositions);
> +    av_freep(&flv->metaVideoColor);
>       return 0;
>   }
>   
> @@ -1028,6 +1085,104 @@ static int resync(AVFormatContext *s)
>       return AVERROR_EOF;
>   }
>   
> +static int flv_parse_video_color_info(AVFormatContext *s, AVStream *st, int64_t next_pos)
> +{
> +    FLVContext *flv = s->priv_data;
> +    AMFDataType type;
> +    AVIOContext *ioc;
> +    char buffer[32];
> +    ioc     = s->pb;
> +
> +    // first object needs to be "colorInfo" string
> +    type = avio_r8(ioc);
> +    if (type != AMF_DATA_TYPE_STRING ||
> +        amf_get_string(ioc, buffer, sizeof(buffer)) < 0)
> +        return TYPE_UNKNOWN;
> +
> +    if (strcmp(buffer, "colorInfo")) {
> +        av_log(s, AV_LOG_DEBUG, "Unknown type %s\n", buffer);
> +        return TYPE_UNKNOWN;
> +    }
> +
> +    flv->metaVideoColor = av_mallocz(sizeof(FLVMetaVideoColor));
> +    if (!flv->metaVideoColor) {
> +        return AVERROR(ENOMEM);
> +    }
> +    flv->meta_color_info_flag = 1;
> +    amf_parse_object(s, NULL, NULL, buffer, next_pos, 0); // parse metadata
> +    return 0;
> +}
> +
> +static int flv_update_video_color_info(AVFormatContext *s, AVStream *st)
> +{
> +    FLVContext *flv = s->priv_data;
> +    const FLVMetaVideoColor* meta_video_color = flv->metaVideoColor;
> +    const FLVMasteringMeta *mastering_meta = &meta_video_color->mastering_meta;
> +
> +    int has_mastering_primaries, has_mastering_luminance;
> +    // Mastering primaries are CIE 1931 coords, and must be > 0.
> +    has_mastering_primaries =
> +        mastering_meta->r_x > 0 && mastering_meta->r_y > 0 &&
> +        mastering_meta->g_x > 0 && mastering_meta->g_y > 0 &&
> +        mastering_meta->b_x > 0 && mastering_meta->b_y > 0 &&
> +        mastering_meta->white_x > 0 && mastering_meta->white_y > 0;
> +    has_mastering_luminance = mastering_meta->max_luminance >= 0 && mastering_meta->min_luminance >= 0;
> +
> +    if (meta_video_color->matrix_coefficients != AVCOL_SPC_RESERVED)
> +        st->codecpar->color_space = meta_video_color->matrix_coefficients;
> +    if (meta_video_color->primaries != AVCOL_PRI_RESERVED &&
> +        meta_video_color->primaries != AVCOL_PRI_RESERVED0)
> +        st->codecpar->color_primaries = meta_video_color->primaries;
> +    if (meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED &&
> +        meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED0)
> +        st->codecpar->color_trc = meta_video_color->transfer_characteristics;
> +
> +    if (meta_video_color->max_cll && meta_video_color->max_fall) {
> +        size_t size = 0;
> +        AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
> +        if (!metadata)
> +            return AVERROR(ENOMEM);
> +        if (!av_packet_side_data_add(&st->codecpar->coded_side_data, &st->codecpar->nb_coded_side_data,
> +                                        AV_PKT_DATA_CONTENT_LIGHT_LEVEL, metadata, size, 0)) {
> +            av_freep(&metadata);
> +            return AVERROR(ENOMEM);
> +        }
> +        metadata->MaxCLL  = meta_video_color->max_cll;
> +        metadata->MaxFALL = meta_video_color->max_fall;
> +    }
> +
> +    if (has_mastering_primaries || has_mastering_luminance) {
> +        AVMasteringDisplayMetadata *metadata;
> +        AVPacketSideData *sd = av_packet_side_data_new(&st->codecpar->coded_side_data,
> +                                                        &st->codecpar->nb_coded_side_data,
> +                                                        AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
> +                                                        sizeof(AVMasteringDisplayMetadata), 0);
> +        if (!sd)
> +            return AVERROR(ENOMEM);
> +        metadata = (AVMasteringDisplayMetadata*)sd->data;
> +        memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
> +        // hdrCll
> +        if (has_mastering_luminance) {
> +            metadata->max_luminance = av_d2q(mastering_meta->max_luminance, INT_MAX);
> +            metadata->min_luminance = av_d2q(mastering_meta->min_luminance, INT_MAX);
> +            metadata->has_luminance = 1;
> +        }
> +        // hdrMdcv
> +        if (has_mastering_primaries) {
> +            metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
> +            metadata->display_primaries[0][1] = av_d2q(mastering_meta->r_y, INT_MAX);
> +            metadata->display_primaries[1][0] = av_d2q(mastering_meta->g_x, INT_MAX);
> +            metadata->display_primaries[1][1] = av_d2q(mastering_meta->g_y, INT_MAX);
> +            metadata->display_primaries[2][0] = av_d2q(mastering_meta->b_x, INT_MAX);
> +            metadata->display_primaries[2][1] = av_d2q(mastering_meta->b_y, INT_MAX);
> +            metadata->white_point[0] = av_d2q(mastering_meta->white_x, INT_MAX);
> +            metadata->white_point[1] = av_d2q(mastering_meta->white_y, INT_MAX);
> +            metadata->has_primaries = 1;
> +        }
> +    }
> +    return 0;
> +}
> +
>   static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
>   {
>       FLVContext *flv = s->priv_data;
> @@ -1100,8 +1255,17 @@ retry:
>               video_codec_id = avio_rb32(s->pb);
>               size -= 4;
>           }
> -        if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD)
> +
> +        if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
> +            type = flags & 0x0F;
> +            if (type == PacketTypeMetadata) {
> +                int ret = flv_parse_video_color_info(s, st, next);
> +                av_log(s, AV_LOG_INFO, "enhanced flv parse metadata ret %d and skip\n", ret);
> +            }
> +            goto skip;
> +        } else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
>               goto skip;
> +        }
>       } else if (type == FLV_TAG_TYPE_META) {
>           stream_type=FLV_STREAM_TYPE_SUBTITLE;
>           if (size > 13 + 1 + 4) { // Header-type metadata stuff
> @@ -1287,6 +1451,11 @@ retry_duration:
>               goto leave;
>           }
>   
> +        if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && flv->meta_color_info_flag) {
> +            flv_update_video_color_info(s, st); // update av packet side data
> +            flv->meta_color_info_flag = 0;
> +        }
> +
>           if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
>               (st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
>               // sign extension

A test for this would be nice. Can be one that creates a file and then 
demuxes it.
See the fate-lavf-fate-* tests to remux an existing sample.
zhupengfei Dec. 13, 2023, 1:41 p.m. UTC | #2 
> 2023年12月5日 05:32，James Almer <jamrial@gmail.com> 写道：
> 
> On 11/15/2023 11:40 AM, zhupengfei via ffmpeg-devel wrote:
>> From: Zhu Pengfei <411294962@qq.com>
>> Signed-off-by: Zhu Pengfei <411294962@qq.com>
>> ---
>>  libavformat/flvdec.c | 171 ++++++++++++++++++++++++++++++++++++++++++-
>>  1 file changed, 170 insertions(+), 1 deletion(-)
>> diff --git a/libavformat/flvdec.c b/libavformat/flvdec.c
>> index e25b5bd163..46bb0825ca 100644
>> --- a/libavformat/flvdec.c
>> +++ b/libavformat/flvdec.c
>> @@ -34,6 +34,7 @@
>>  #include "libavutil/intfloat.h"
>>  #include "libavutil/intreadwrite.h"
>>  #include "libavutil/mathematics.h"
>> +#include "libavutil/mastering_display_metadata.h"
>>  #include "avformat.h"
>>  #include "demux.h"
>>  #include "internal.h"
>> @@ -45,6 +46,28 @@
>>    #define MAX_DEPTH 16      ///< arbitrary limit to prevent unbounded recursion
>>  +typedef struct FLVMasteringMeta {
>> +    double r_x;
>> +    double r_y;
>> +    double g_x;
>> +    double g_y;
>> +    double b_x;
>> +    double b_y;
>> +    double white_x;
>> +    double white_y;
>> +    double max_luminance;
>> +    double min_luminance;
>> +} FLVMasteringMeta;
>> +
>> +typedef struct FLVMetaVideoColor {
>> +    uint64_t matrix_coefficients;
>> +    uint64_t transfer_characteristics;
>> +    uint64_t primaries;
>> +    uint64_t max_cll;
>> +    uint64_t max_fall;
>> +    FLVMasteringMeta mastering_meta;
>> +} FLVMetaVideoColor;
>> +
>>  typedef struct FLVContext {
>>      const AVClass *class; ///< Class for private options.
>>      int trust_metadata;   ///< configure streams according onMetaData
>> @@ -80,6 +103,8 @@ typedef struct FLVContext {
>>      int64_t time_offset;
>>      int64_t time_pos;
>>  +    FLVMetaVideoColor *metaVideoColor;
>> +    int meta_color_info_flag;
>>  } FLVContext;
>>    /* AMF date type */
>> @@ -524,6 +549,7 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream,
>>      FLVContext *flv = s->priv_data;
>>      AVIOContext *ioc;
>>      AMFDataType amf_type;
>> +    FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
>>      char str_val[1024];
>>      double num_val;
>>      amf_date date;
>> @@ -655,6 +681,36 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream,
>>                      } else if (!strcmp(key, "height") && vpar) {
>>                          vpar->height = num_val;
>>                      }
>> +                } else if (!strcmp(key, "colorPrimaries") && meta_video_color) {
> 
> You should put this inside an "else if (meta_video_color)" block, instead of checking for meta_video_color with every single string.
> 
> See how it's done for flv->trust_metadata above.
> 
>> +                    meta_video_color->primaries = num_val;
>> +                } else if (!strcmp(key, "transferCharacteristics") && meta_video_color) {
>> +                    meta_video_color->transfer_characteristics = num_val;
>> +                } else if (!strcmp(key, "matrixCoefficients") && meta_video_color) {
>> +                    meta_video_color->matrix_coefficients = num_val;
>> +                } else if (!strcmp(key, "maxFall") && meta_video_color) {
>> +                    meta_video_color->max_fall = num_val;
>> +                } else if (!strcmp(key, "maxCLL") && meta_video_color) {
>> +                    meta_video_color->max_cll = num_val;
>> +                } else if (!strcmp(key, "redX") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.r_x = num_val;
>> +                } else if (!strcmp(key, "redY") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.r_y = num_val;
>> +                } else if (!strcmp(key, "greenX") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.g_x = num_val;
>> +                } else if (!strcmp(key, "greenY") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.g_y = num_val;
>> +                } else if (!strcmp(key, "blueX") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.b_x = num_val;
>> +                } else if (!strcmp(key, "blueY") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.b_y = num_val;
>> +                } else if (!strcmp(key, "whitePointX") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.white_x = num_val;
>> +                } else if (!strcmp(key, "whitePointY") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.white_y = num_val;
>> +                } else if (!strcmp(key, "maxLuminance") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.max_luminance = num_val;
>> +                } else if (!strcmp(key, "minLuminance") && meta_video_color) {
>> +                    meta_video_color->mastering_meta.min_luminance = num_val;
>>                  }
>>              }
>>              if (amf_type == AMF_DATA_TYPE_STRING) {
>> @@ -824,6 +880,7 @@ static int flv_read_close(AVFormatContext *s)
>>          av_freep(&flv->new_extradata[i]);
>>      av_freep(&flv->keyframe_times);
>>      av_freep(&flv->keyframe_filepositions);
>> +    av_freep(&flv->metaVideoColor);
>>      return 0;
>>  }
>>  @@ -1028,6 +1085,104 @@ static int resync(AVFormatContext *s)
>>      return AVERROR_EOF;
>>  }
>>  +static int flv_parse_video_color_info(AVFormatContext *s, AVStream *st, int64_t next_pos)
>> +{
>> +    FLVContext *flv = s->priv_data;
>> +    AMFDataType type;
>> +    AVIOContext *ioc;
>> +    char buffer[32];
>> +    ioc     = s->pb;
>> +
>> +    // first object needs to be "colorInfo" string
>> +    type = avio_r8(ioc);
>> +    if (type != AMF_DATA_TYPE_STRING ||
>> +        amf_get_string(ioc, buffer, sizeof(buffer)) < 0)
>> +        return TYPE_UNKNOWN;
>> +
>> +    if (strcmp(buffer, "colorInfo")) {
>> +        av_log(s, AV_LOG_DEBUG, "Unknown type %s\n", buffer);
>> +        return TYPE_UNKNOWN;
>> +    }
>> +
>> +    flv->metaVideoColor = av_mallocz(sizeof(FLVMetaVideoColor));
>> +    if (!flv->metaVideoColor) {
>> +        return AVERROR(ENOMEM);
>> +    }
>> +    flv->meta_color_info_flag = 1;
>> +    amf_parse_object(s, NULL, NULL, buffer, next_pos, 0); // parse metadata
>> +    return 0;
>> +}
>> +
>> +static int flv_update_video_color_info(AVFormatContext *s, AVStream *st)
>> +{
>> +    FLVContext *flv = s->priv_data;
>> +    const FLVMetaVideoColor* meta_video_color = flv->metaVideoColor;
>> +    const FLVMasteringMeta *mastering_meta = &meta_video_color->mastering_meta;
>> +
>> +    int has_mastering_primaries, has_mastering_luminance;
>> +    // Mastering primaries are CIE 1931 coords, and must be > 0.
>> +    has_mastering_primaries =
>> +        mastering_meta->r_x > 0 && mastering_meta->r_y > 0 &&
>> +        mastering_meta->g_x > 0 && mastering_meta->g_y > 0 &&
>> +        mastering_meta->b_x > 0 && mastering_meta->b_y > 0 &&
>> +        mastering_meta->white_x > 0 && mastering_meta->white_y > 0;
>> +    has_mastering_luminance = mastering_meta->max_luminance >= 0 && mastering_meta->min_luminance >= 0;
>> +
>> +    if (meta_video_color->matrix_coefficients != AVCOL_SPC_RESERVED)
>> +        st->codecpar->color_space = meta_video_color->matrix_coefficients;
>> +    if (meta_video_color->primaries != AVCOL_PRI_RESERVED &&
>> +        meta_video_color->primaries != AVCOL_PRI_RESERVED0)
>> +        st->codecpar->color_primaries = meta_video_color->primaries;
>> +    if (meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED &&
>> +        meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED0)
>> +        st->codecpar->color_trc = meta_video_color->transfer_characteristics;
>> +
>> +    if (meta_video_color->max_cll && meta_video_color->max_fall) {
>> +        size_t size = 0;
>> +        AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
>> +        if (!metadata)
>> +            return AVERROR(ENOMEM);
>> +        if (!av_packet_side_data_add(&st->codecpar->coded_side_data, &st->codecpar->nb_coded_side_data,
>> +                                        AV_PKT_DATA_CONTENT_LIGHT_LEVEL, metadata, size, 0)) {
>> +            av_freep(&metadata);
>> +            return AVERROR(ENOMEM);
>> +        }
>> +        metadata->MaxCLL  = meta_video_color->max_cll;
>> +        metadata->MaxFALL = meta_video_color->max_fall;
>> +    }
>> +
>> +    if (has_mastering_primaries || has_mastering_luminance) {
>> +        AVMasteringDisplayMetadata *metadata;
>> +        AVPacketSideData *sd = av_packet_side_data_new(&st->codecpar->coded_side_data,
>> +                                                        &st->codecpar->nb_coded_side_data,
>> +                                                        AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
>> +                                                        sizeof(AVMasteringDisplayMetadata), 0);
>> +        if (!sd)
>> +            return AVERROR(ENOMEM);
>> +        metadata = (AVMasteringDisplayMetadata*)sd->data;
>> +        memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
>> +        // hdrCll
>> +        if (has_mastering_luminance) {
>> +            metadata->max_luminance = av_d2q(mastering_meta->max_luminance, INT_MAX);
>> +            metadata->min_luminance = av_d2q(mastering_meta->min_luminance, INT_MAX);
>> +            metadata->has_luminance = 1;
>> +        }
>> +        // hdrMdcv
>> +        if (has_mastering_primaries) {
>> +            metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
>> +            metadata->display_primaries[0][1] = av_d2q(mastering_meta->r_y, INT_MAX);
>> +            metadata->display_primaries[1][0] = av_d2q(mastering_meta->g_x, INT_MAX);
>> +            metadata->display_primaries[1][1] = av_d2q(mastering_meta->g_y, INT_MAX);
>> +            metadata->display_primaries[2][0] = av_d2q(mastering_meta->b_x, INT_MAX);
>> +            metadata->display_primaries[2][1] = av_d2q(mastering_meta->b_y, INT_MAX);
>> +            metadata->white_point[0] = av_d2q(mastering_meta->white_x, INT_MAX);
>> +            metadata->white_point[1] = av_d2q(mastering_meta->white_y, INT_MAX);
>> +            metadata->has_primaries = 1;
>> +        }
>> +    }
>> +    return 0;
>> +}
>> +
>>  static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
>>  {
>>      FLVContext *flv = s->priv_data;
>> @@ -1100,8 +1255,17 @@ retry:
>>              video_codec_id = avio_rb32(s->pb);
>>              size -= 4;
>>          }
>> -        if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD)
>> +
>> +        if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
>> +            type = flags & 0x0F;
>> +            if (type == PacketTypeMetadata) {
>> +                int ret = flv_parse_video_color_info(s, st, next);
>> +                av_log(s, AV_LOG_INFO, "enhanced flv parse metadata ret %d and skip\n", ret);
>> +            }
>> +            goto skip;
>> +        } else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
>>              goto skip;
>> +        }
>>      } else if (type == FLV_TAG_TYPE_META) {
>>          stream_type=FLV_STREAM_TYPE_SUBTITLE;
>>          if (size > 13 + 1 + 4) { // Header-type metadata stuff
>> @@ -1287,6 +1451,11 @@ retry_duration:
>>              goto leave;
>>          }
>>  +        if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && flv->meta_color_info_flag) {
>> +            flv_update_video_color_info(s, st); // update av packet side data
>> +            flv->meta_color_info_flag = 0;
>> +        }
>> +
>>          if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
>>              (st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
>>              // sign extension
> 
> A test for this would be nice. Can be one that creates a file and then demuxes it.
> See the fate-lavf-fate-* tests to remux an existing sample.


Thank you very much for your suggestion.   All other comments have been modified as per your suggestions locally. 

But about fate test, I have some questions that would like to ask for advice:
1. During the development stage, can I use ffmpeg with the mov/mkv muxer to write side data to construct an hdr flv file (ffmpeg -i *** -metadata:s:v **.mkv)
2. Referring to fate-lavf-fate-* tests, I did not find sample file with metadata in the official fate suite. If test-fate uses codec copy, it cannot obtain sidedata and therefore cannot be constructed flv file.
3. Do you have any suggestions? Besides creating a new fate suite file, are there any other methods?

Thanks


> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org <mailto:ffmpeg-devel@ffmpeg.org>
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org <mailto:ffmpeg-devel-request@ffmpeg.org> with subject "unsubscribe".
diff mbox series

Patch

diff --git a/libavformat/flvdec.c b/libavformat/flvdec.c
index e25b5bd163..46bb0825ca 100644
--- a/libavformat/flvdec.c
+++ b/libavformat/flvdec.c
@@ -34,6 +34,7 @@ 
 #include "libavutil/intfloat.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/mathematics.h"
+#include "libavutil/mastering_display_metadata.h"
 #include "avformat.h"
 #include "demux.h"
 #include "internal.h"
@@ -45,6 +46,28 @@ 
 
 #define MAX_DEPTH 16      ///< arbitrary limit to prevent unbounded recursion
 
+typedef struct FLVMasteringMeta {
+    double r_x;
+    double r_y;
+    double g_x;
+    double g_y;
+    double b_x;
+    double b_y;
+    double white_x;
+    double white_y;
+    double max_luminance;
+    double min_luminance;
+} FLVMasteringMeta;
+
+typedef struct FLVMetaVideoColor {
+    uint64_t matrix_coefficients;
+    uint64_t transfer_characteristics;
+    uint64_t primaries;
+    uint64_t max_cll;
+    uint64_t max_fall;
+    FLVMasteringMeta mastering_meta;
+} FLVMetaVideoColor;
+
 typedef struct FLVContext {
     const AVClass *class; ///< Class for private options.
     int trust_metadata;   ///< configure streams according onMetaData
@@ -80,6 +103,8 @@  typedef struct FLVContext {
     int64_t time_offset;
     int64_t time_pos;
 
+    FLVMetaVideoColor *metaVideoColor;
+    int meta_color_info_flag;
 } FLVContext;
 
 /* AMF date type */
@@ -524,6 +549,7 @@  static int amf_parse_object(AVFormatContext *s, AVStream *astream,
     FLVContext *flv = s->priv_data;
     AVIOContext *ioc;
     AMFDataType amf_type;
+    FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
     char str_val[1024];
     double num_val;
     amf_date date;
@@ -655,6 +681,36 @@  static int amf_parse_object(AVFormatContext *s, AVStream *astream,
                     } else if (!strcmp(key, "height") && vpar) {
                         vpar->height = num_val;
                     }
+                } else if (!strcmp(key, "colorPrimaries") && meta_video_color) {
+                    meta_video_color->primaries = num_val;
+                } else if (!strcmp(key, "transferCharacteristics") && meta_video_color) {
+                    meta_video_color->transfer_characteristics = num_val;
+                } else if (!strcmp(key, "matrixCoefficients") && meta_video_color) {
+                    meta_video_color->matrix_coefficients = num_val;
+                } else if (!strcmp(key, "maxFall") && meta_video_color) {
+                    meta_video_color->max_fall = num_val;
+                } else if (!strcmp(key, "maxCLL") && meta_video_color) {
+                    meta_video_color->max_cll = num_val;
+                } else if (!strcmp(key, "redX") && meta_video_color) {
+                    meta_video_color->mastering_meta.r_x = num_val;
+                } else if (!strcmp(key, "redY") && meta_video_color) {
+                    meta_video_color->mastering_meta.r_y = num_val;
+                } else if (!strcmp(key, "greenX") && meta_video_color) {
+                    meta_video_color->mastering_meta.g_x = num_val;
+                } else if (!strcmp(key, "greenY") && meta_video_color) {
+                    meta_video_color->mastering_meta.g_y = num_val;
+                } else if (!strcmp(key, "blueX") && meta_video_color) {
+                    meta_video_color->mastering_meta.b_x = num_val;
+                } else if (!strcmp(key, "blueY") && meta_video_color) {
+                    meta_video_color->mastering_meta.b_y = num_val;
+                } else if (!strcmp(key, "whitePointX") && meta_video_color) {
+                    meta_video_color->mastering_meta.white_x = num_val;
+                } else if (!strcmp(key, "whitePointY") && meta_video_color) {
+                    meta_video_color->mastering_meta.white_y = num_val;
+                } else if (!strcmp(key, "maxLuminance") && meta_video_color) {
+                    meta_video_color->mastering_meta.max_luminance = num_val;
+                } else if (!strcmp(key, "minLuminance") && meta_video_color) {
+                    meta_video_color->mastering_meta.min_luminance = num_val;
                 }
             }
             if (amf_type == AMF_DATA_TYPE_STRING) {
@@ -824,6 +880,7 @@  static int flv_read_close(AVFormatContext *s)
         av_freep(&flv->new_extradata[i]);
     av_freep(&flv->keyframe_times);
     av_freep(&flv->keyframe_filepositions);
+    av_freep(&flv->metaVideoColor);
     return 0;
 }
 
@@ -1028,6 +1085,104 @@  static int resync(AVFormatContext *s)
     return AVERROR_EOF;
 }
 
+static int flv_parse_video_color_info(AVFormatContext *s, AVStream *st, int64_t next_pos)
+{
+    FLVContext *flv = s->priv_data;
+    AMFDataType type;
+    AVIOContext *ioc;
+    char buffer[32];
+    ioc     = s->pb;
+
+    // first object needs to be "colorInfo" string
+    type = avio_r8(ioc);
+    if (type != AMF_DATA_TYPE_STRING ||
+        amf_get_string(ioc, buffer, sizeof(buffer)) < 0)
+        return TYPE_UNKNOWN;
+
+    if (strcmp(buffer, "colorInfo")) {
+        av_log(s, AV_LOG_DEBUG, "Unknown type %s\n", buffer);
+        return TYPE_UNKNOWN;
+    }
+
+    flv->metaVideoColor = av_mallocz(sizeof(FLVMetaVideoColor));
+    if (!flv->metaVideoColor) {
+        return AVERROR(ENOMEM);
+    }
+    flv->meta_color_info_flag = 1;
+    amf_parse_object(s, NULL, NULL, buffer, next_pos, 0); // parse metadata
+    return 0;
+}
+
+static int flv_update_video_color_info(AVFormatContext *s, AVStream *st)
+{
+    FLVContext *flv = s->priv_data;
+    const FLVMetaVideoColor* meta_video_color = flv->metaVideoColor;
+    const FLVMasteringMeta *mastering_meta = &meta_video_color->mastering_meta;
+
+    int has_mastering_primaries, has_mastering_luminance;
+    // Mastering primaries are CIE 1931 coords, and must be > 0.
+    has_mastering_primaries =
+        mastering_meta->r_x > 0 && mastering_meta->r_y > 0 &&
+        mastering_meta->g_x > 0 && mastering_meta->g_y > 0 &&
+        mastering_meta->b_x > 0 && mastering_meta->b_y > 0 &&
+        mastering_meta->white_x > 0 && mastering_meta->white_y > 0;
+    has_mastering_luminance = mastering_meta->max_luminance >= 0 && mastering_meta->min_luminance >= 0;
+
+    if (meta_video_color->matrix_coefficients != AVCOL_SPC_RESERVED)
+        st->codecpar->color_space = meta_video_color->matrix_coefficients;
+    if (meta_video_color->primaries != AVCOL_PRI_RESERVED &&
+        meta_video_color->primaries != AVCOL_PRI_RESERVED0)
+        st->codecpar->color_primaries = meta_video_color->primaries;
+    if (meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED &&
+        meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED0)
+        st->codecpar->color_trc = meta_video_color->transfer_characteristics;
+
+    if (meta_video_color->max_cll && meta_video_color->max_fall) {
+        size_t size = 0;
+        AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
+        if (!metadata)
+            return AVERROR(ENOMEM);
+        if (!av_packet_side_data_add(&st->codecpar->coded_side_data, &st->codecpar->nb_coded_side_data,
+                                        AV_PKT_DATA_CONTENT_LIGHT_LEVEL, metadata, size, 0)) {
+            av_freep(&metadata);
+            return AVERROR(ENOMEM);
+        }
+        metadata->MaxCLL  = meta_video_color->max_cll;
+        metadata->MaxFALL = meta_video_color->max_fall;
+    }
+
+    if (has_mastering_primaries || has_mastering_luminance) {
+        AVMasteringDisplayMetadata *metadata;
+        AVPacketSideData *sd = av_packet_side_data_new(&st->codecpar->coded_side_data,
+                                                        &st->codecpar->nb_coded_side_data,
+                                                        AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
+                                                        sizeof(AVMasteringDisplayMetadata), 0);
+        if (!sd)
+            return AVERROR(ENOMEM);
+        metadata = (AVMasteringDisplayMetadata*)sd->data;
+        memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
+        // hdrCll
+        if (has_mastering_luminance) {
+            metadata->max_luminance = av_d2q(mastering_meta->max_luminance, INT_MAX);
+            metadata->min_luminance = av_d2q(mastering_meta->min_luminance, INT_MAX);
+            metadata->has_luminance = 1;
+        }
+        // hdrMdcv
+        if (has_mastering_primaries) {
+            metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
+            metadata->display_primaries[0][1] = av_d2q(mastering_meta->r_y, INT_MAX);
+            metadata->display_primaries[1][0] = av_d2q(mastering_meta->g_x, INT_MAX);
+            metadata->display_primaries[1][1] = av_d2q(mastering_meta->g_y, INT_MAX);
+            metadata->display_primaries[2][0] = av_d2q(mastering_meta->b_x, INT_MAX);
+            metadata->display_primaries[2][1] = av_d2q(mastering_meta->b_y, INT_MAX);
+            metadata->white_point[0] = av_d2q(mastering_meta->white_x, INT_MAX);
+            metadata->white_point[1] = av_d2q(mastering_meta->white_y, INT_MAX);
+            metadata->has_primaries = 1;
+        }
+    }
+    return 0;
+}
+
 static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
     FLVContext *flv = s->priv_data;
@@ -1100,8 +1255,17 @@  retry:
             video_codec_id = avio_rb32(s->pb);
             size -= 4;
         }
-        if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD)
+
+        if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
+            type = flags & 0x0F;
+            if (type == PacketTypeMetadata) {
+                int ret = flv_parse_video_color_info(s, st, next);
+                av_log(s, AV_LOG_INFO, "enhanced flv parse metadata ret %d and skip\n", ret);
+            }
+            goto skip;
+        } else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
             goto skip;
+        }
     } else if (type == FLV_TAG_TYPE_META) {
         stream_type=FLV_STREAM_TYPE_SUBTITLE;
         if (size > 13 + 1 + 4) { // Header-type metadata stuff
@@ -1287,6 +1451,11 @@  retry_duration:
             goto leave;
         }
 
+        if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && flv->meta_color_info_flag) {
+            flv_update_video_color_info(s, st); // update av packet side data
+            flv->meta_color_info_flag = 0;
+        }
+
         if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
             (st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
             // sign extension