[FFmpeg-devel,v2,2/3] avcodec/nvenc: add master display and light level sei for HDR10

Submitted by lance.lmwang@gmail.com on May 28, 2019, 3:29 a.m.

Details

Message ID 20190528032953.3864-2-lance.lmwang@gmail.com
State New
Headers show

Commit Message

lance.lmwang@gmail.com May 28, 2019, 3:29 a.m.
From: Limin Wang <lance.lmwang@gmail.com>

The testing command for the HDR10 output with nvenc:
$ ./ffmpeg_g -y -i 4K.mp4 -c:v hevc_nvenc -color_primaries bt2020 -colorspace bt2020_ncl -color_trc smpte2084 -sei hdr10 \
        -master_display "G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(10000000,50)" -max_cll "0, 0" test.ts

Please notice it is preferable to use the frame sei side data than master_display and max_cll paramters config

Signed-off-by: Limin Wang <lance.lmwang@gmail.com>
---
 libavcodec/nvenc.c      | 114 ++++++++++++++++++++++++++++++++++++++++
 libavcodec/nvenc.h      |  18 +++++++
 libavcodec/nvenc_hevc.c |  11 ++++
 3 files changed, 143 insertions(+)

Patch hide | download patch | download mbox

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index 75dda6d689..30cf4df928 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -22,6 +22,8 @@ 
 #include "config.h"
 
 #include "nvenc.h"
+#include "hevc_sei.h"
+#include "put_bits.h"
 
 #include "libavutil/hwcontext_cuda.h"
 #include "libavutil/hwcontext.h"
@@ -30,6 +32,7 @@ 
 #include "libavutil/avassert.h"
 #include "libavutil/mem.h"
 #include "libavutil/pixdesc.h"
+#include "libavutil/mastering_display_metadata.h"
 #include "internal.h"
 
 #define CHECK_CU(x) FF_CUDA_CHECK_DL(avctx, dl_fn->cuda_dl, x)
@@ -1491,6 +1494,46 @@  av_cold int ff_nvenc_encode_init(AVCodecContext *avctx)
         ctx->data_pix_fmt = avctx->pix_fmt;
     }
 
+    ctx->display_primaries_x[0] = 13250;
+    ctx->display_primaries_y[0] = 34500;
+    ctx->display_primaries_x[1] = 7500;
+    ctx->display_primaries_y[1] = 3000;
+    ctx->display_primaries_x[2] = 34000;
+    ctx->display_primaries_y[2] = 16000;
+    ctx->white_point_x          = 15635;
+    ctx->white_point_y          = 16450;
+    ctx->max_display_mastering_luminance = 10000000;
+    ctx->min_display_mastering_luminance = 500;
+    ctx->max_content_light_level = 0;
+    ctx->max_pic_average_light_level = 0;
+    if (ctx->master_display) {
+        ret = sscanf(ctx->master_display, "G(%hu,%hu)B(%hu,%hu)R(%hu,%hu)WP(%hu,%hu)L(%u,%u)",
+                &ctx->display_primaries_x[0], &ctx->display_primaries_y[0],
+                &ctx->display_primaries_x[1], &ctx->display_primaries_y[1],
+                &ctx->display_primaries_x[2], &ctx->display_primaries_y[2],
+                &ctx->white_point_x, &ctx->white_point_y,
+                &ctx->max_display_mastering_luminance, &ctx->min_display_mastering_luminance);
+        if (ret != 10) {
+            ret = sscanf(ctx->master_display, "G[%hu,%hu]B[%hu,%hu]R[%hu,%hu]WP[%hu,%hu]L[%u,%u]",
+                &ctx->display_primaries_x[0], &ctx->display_primaries_y[0],
+                &ctx->display_primaries_x[1], &ctx->display_primaries_y[1],
+                &ctx->display_primaries_x[2], &ctx->display_primaries_y[2],
+                &ctx->white_point_x, &ctx->white_point_y,
+                &ctx->max_display_mastering_luminance, &ctx->min_display_mastering_luminance);
+        }
+
+        if (ret != 10) {
+            av_log(avctx, AV_LOG_INFO, "Failed to parse master display(%s)\n", ctx->master_display);
+        }
+    }
+
+    if (ctx->max_cll) {
+        ret = sscanf(ctx->max_cll, "%hu,%hu", &ctx->max_content_light_level, &ctx->max_pic_average_light_level);
+        if (ret != 2) {
+            av_log(avctx, AV_LOG_INFO, "Failed to parse max cll(%s)\n", ctx->max_cll);
+        }
+    }
+
     if ((ret = nvenc_load_libraries(avctx)) < 0)
         return ret;
 
@@ -2110,6 +2153,77 @@  int ff_nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
             }
         }
 
+        if (ctx->sei  & SEI_MASTERING_DISPLAY) {
+            AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA);
+
+            if (sd) {
+                AVMasteringDisplayMetadata *mdm = (AVMasteringDisplayMetadata *)sd->data;
+                // HEVC uses a g,b,r ordering, which we convert from a more natural r,g,b
+                const int mapping[3] = {2, 0, 1};
+                const int chroma_den = 50000;
+                const int luma_den = 10000;
+
+                if (mdm->has_primaries && mdm->has_luminance) {
+
+                    for (i = 0; i < 3; i++) {
+                        const int j = mapping[i];
+                        ctx->display_primaries_x[i] = chroma_den * av_q2d(mdm->display_primaries[j][0]);
+                        ctx->display_primaries_y[i] = chroma_den * av_q2d( mdm->display_primaries[j][1]);
+                    }
+
+                    ctx->white_point_x = chroma_den * av_q2d(mdm->white_point[0]);
+                    ctx->white_point_y = chroma_den * av_q2d(mdm->white_point[1]);
+                    ctx->max_display_mastering_luminance = luma_den * av_q2d(mdm->max_luminance);
+                    ctx->min_display_mastering_luminance = luma_den * av_q2d(mdm->min_luminance);
+                }
+            }
+
+            sei_data[sei_count].payloadSize = 24;
+            sei_data[sei_count].payloadType = HEVC_SEI_TYPE_MASTERING_DISPLAY_INFO;
+            sei_data[sei_count].payload     = av_mallocz(sei_data[sei_count].payloadSize);
+            if (sei_data[sei_count].payload) {
+                PutBitContext pb;
+
+                init_put_bits(&pb, sei_data[sei_count].payload, sei_data[sei_count].payloadSize);
+                for (i = 0; i < 3; i++) {
+                    put_bits(&pb, 16, ctx->display_primaries_x[i]);
+                    put_bits(&pb, 16, ctx->display_primaries_y[i]);
+                }
+                put_bits(&pb, 16, ctx->white_point_x);
+                put_bits(&pb, 16, ctx->white_point_y);
+                put_bits(&pb, 32, ctx->max_display_mastering_luminance);
+                put_bits(&pb, 32, ctx->min_display_mastering_luminance);
+                flush_put_bits(&pb);
+
+                sei_count ++;
+            }
+        }
+
+        if (ctx->sei & SEI_CONTENT_LIGHT_LEVEL) {
+            AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL);
+
+            if (sd) {
+                AVContentLightMetadata *clm = (AVContentLightMetadata *)sd->data;
+
+                ctx->max_content_light_level     = FFMIN(clm->MaxCLL,  65535);
+                ctx->max_pic_average_light_level = FFMIN(clm->MaxFALL, 65535);
+            }
+
+            sei_data[sei_count].payloadSize = 4;
+            sei_data[sei_count].payloadType = HEVC_SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO;
+            sei_data[sei_count].payload     = av_mallocz(sei_data[sei_count].payloadSize);
+            if (sei_data[sei_count].payload) {
+                PutBitContext pb;
+
+                init_put_bits(&pb, sei_data[sei_count].payload, sei_data[sei_count].payloadSize);
+                put_bits(&pb, 16, ctx->max_content_light_level);
+                put_bits(&pb, 16, ctx->max_pic_average_light_level);
+                flush_put_bits(&pb);
+
+                sei_count ++;
+            }
+        }
+
         nvenc_codec_specific_pic_params(avctx, &pic_params, sei_data, sei_count);
     } else {
         pic_params.encodePicFlags = NV_ENC_PIC_FLAG_EOS;
diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
index ddd6168409..583c48d090 100644
--- a/libavcodec/nvenc.h
+++ b/libavcodec/nvenc.h
@@ -54,6 +54,11 @@  typedef void ID3D11Device;
 #define NVENC_HAVE_HEVC_BFRAME_REF_MODE
 #endif
 
+enum {
+    SEI_MASTERING_DISPLAY       = 0x08,
+    SEI_CONTENT_LIGHT_LEVEL     = 0x10,
+};
+
 typedef struct NvencSurface
 {
     NV_ENC_INPUT_PTR input_surface;
@@ -192,6 +197,19 @@  typedef struct NvencContext
     int coder;
     int b_ref_mode;
     int a53_cc;
+    uint64_t sei;
+
+    char *master_display;
+    char *max_cll;
+    uint16_t display_primaries_x[3];
+    uint16_t display_primaries_y[3];
+    uint16_t white_point_x;
+    uint16_t white_point_y;
+    uint32_t max_display_mastering_luminance;
+    uint32_t min_display_mastering_luminance;
+
+    uint16_t max_content_light_level;
+    uint16_t max_pic_average_light_level;
 } NvencContext;
 
 int ff_nvenc_encode_init(AVCodecContext *avctx);
diff --git a/libavcodec/nvenc_hevc.c b/libavcodec/nvenc_hevc.c
index d567d960ba..74ebd03d8e 100644
--- a/libavcodec/nvenc_hevc.c
+++ b/libavcodec/nvenc_hevc.c
@@ -127,6 +127,17 @@  static const AVOption options[] = {
     { "each",         "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0,       VE, "b_ref_mode" },
     { "middle",       "",                                   0,                    AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0,       VE, "b_ref_mode" },
 #endif
+    { "sei", "Set SEI to include",
+                                                            OFFSET(sei), AV_OPT_TYPE_FLAGS,
+                                                            { .i64 = SEI_MASTERING_DISPLAY | SEI_CONTENT_LIGHT_LEVEL },
+                                                            0, INT_MAX, VE, "sei" },
+    { "hdr10","Include HDR metadata for mastering display colour volume and content light level information",
+                                                            0, AV_OPT_TYPE_CONST,   { .i64 = SEI_MASTERING_DISPLAY | SEI_CONTENT_LIGHT_LEVEL },
+                                                            0, 0, VE, "sei" },
+    { "master_display",     "SMPTE ST 2086 master display color volume info SEI (HDR), the string format is: \"G(x,y)B(x,y)R(x,y)WP(x,y)L(max,min)\"",
+                                                            OFFSET(master_display), AV_OPT_TYPE_STRING,   { .str = NULL }, 0, 0, VE },
+    { "max_cll",             "content light level info, the string format is: \"cll, fall\"",
+                                                            OFFSET(max_cll), AV_OPT_TYPE_STRING,   { .str = NULL }, 0, 0, VE },
     { NULL }
 };