diff mbox series

[FFmpeg-devel,11/11] avcodec/encode: enforce AVCodec capabilities at encode time

Message ID 20240112082950.41637-12-ffmpeg@haasn.xyz
State New
Headers show
Series YUVJ removal | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Niklas Haas Jan. 12, 2024, 8:26 a.m. UTC
From: Niklas Haas <git@haasn.dev>

In general, the logic is always the same: if the codec supports only a
single format, enforce it if possible. Otherwise, throw an error when
an incompatible format is passed.

To preserve backwards compatibility and make this check less pedantic
than it needs to be, always consider 'AVCOL_*_UNSPECIFIED' a valid
value. That way, we will only throw an error if the user explicitly set
a color value that we explicitly know is unsupported.
---
 libavcodec/encode.c | 59 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
diff mbox series

Patch

diff --git a/libavcodec/encode.c b/libavcodec/encode.c
index 31565638fa..32eb7dd9eb 100644
--- a/libavcodec/encode.c
+++ b/libavcodec/encode.c
@@ -574,10 +574,23 @@  int attribute_align_arg avcodec_receive_packet(AVCodecContext *avctx, AVPacket *
     return 0;
 }
 
+static int fmt_is_regular_yuv(enum AVPixelFormat fmt)
+{
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
+    if (!desc || desc->nb_components < 3)
+        return 0;
+    if (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL |
+                       AV_PIX_FMT_FLAG_XYZ | AV_PIX_FMT_FLAG_FLOAT))
+        return 0;
+    return 1;
+}
+
+
 static int encode_preinit_video(AVCodecContext *avctx)
 {
     const AVCodec *c = avctx->codec;
     const AVPixFmtDescriptor *pixdesc = av_pix_fmt_desc_get(avctx->pix_fmt);
+    enum AVPixelFormat swfmt = avctx->pix_fmt;
     int i;
 
     if (!av_get_pix_fmt_name(avctx->pix_fmt)) {
@@ -605,6 +618,52 @@  static int encode_preinit_video(AVCodecContext *avctx)
         }
     }
 
+    if (pixdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
+        const AVHWFramesContext *hwfc = (void *) avctx->hw_frames_ctx->data;
+        swfmt = hwfc->sw_format;
+    }
+
+    if (fmt_is_regular_yuv(swfmt)) {
+        if (avctx->colorspace != AVCOL_SPC_UNSPECIFIED && c->color_spaces) {
+            for (i = 0; c->color_spaces[i] != AVCOL_SPC_UNSPECIFIED; i++)
+                if (avctx->colorspace == c->color_spaces[i])
+                    break;
+            if (c->color_spaces[i] == AVCOL_SPC_UNSPECIFIED) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "Specified color space %s is not supported by the %s encoder.\n",
+                       av_color_space_name(avctx->colorspace), c->name);
+                av_log(avctx, AV_LOG_ERROR, "Supported color spaces:\n");
+                for (int p = 0; c->color_spaces[p] != AVCOL_SPC_UNSPECIFIED; p++) {
+                    av_log(avctx, AV_LOG_ERROR, "  %s\n",
+                           av_color_space_name(c->color_spaces[p]));
+                }
+                return AVERROR(EINVAL);
+            }
+        }
+
+        if (avctx->color_range != AVCOL_RANGE_UNSPECIFIED && c->color_ranges) {
+            for (i = 0; c->color_ranges[i] != AVCOL_RANGE_UNSPECIFIED; i++) {
+                if (avctx->color_range == c->color_ranges[i])
+                    break;
+            }
+            if (c->color_ranges[i] == AVCOL_RANGE_UNSPECIFIED) {
+                if (i == 1 && !avctx->color_range) {
+                    avctx->color_range = c->color_ranges[0];
+                } else if (avctx->color_range) {
+                    av_log(avctx, AV_LOG_ERROR,
+                           "Specified color range %s is not supported by the %s encoder.\n",
+                           av_color_range_name(avctx->color_range), c->name);
+                    av_log(avctx, AV_LOG_ERROR, "Supported color ranges:\n");
+                    for (int p = 0; c->color_ranges[p] != AVCOL_RANGE_UNSPECIFIED; p++) {
+                        av_log(avctx, AV_LOG_ERROR, "  %s\n",
+                               av_color_range_name(c->color_ranges[p]));
+                    }
+                    return AVERROR(EINVAL);
+                }
+            }
+        }
+    }
+
     if (    avctx->bits_per_raw_sample < 0
         || (avctx->bits_per_raw_sample > 8 && pixdesc->comp[0].depth <= 8)) {
         av_log(avctx, AV_LOG_WARNING, "Specified bit depth %d not possible with the specified pixel formats depth %d\n",