Message ID | 20220323110325.5499-3-leo.izen@gmail.com |
---|---|
State | New |
Headers | show |
Series | [FFmpeg-devel,v9,1/5] avcodec/jpegxl: add Jpeg XL image codec and parser | expand |
Context | Check | Description |
---|---|---|
yinshiyou/make_loongarch64 | success | Make finished |
yinshiyou/make_fate_loongarch64 | success | Make fate finished |
andriy/make_x86 | success | Make finished |
andriy/make_fate_x86 | success | Make fate finished |
Leo Izen: > This commit adds encoding support to libavcodec > for Jpeg XL images via the external library libjxl. > --- > configure | 3 +- > libavcodec/Makefile | 1 + > libavcodec/allcodecs.c | 1 + > libavcodec/libjxlenc.c | 385 +++++++++++++++++++++++++++++++++++++++++ > 4 files changed, 389 insertions(+), 1 deletion(-) > create mode 100644 libavcodec/libjxlenc.c > > diff --git a/configure b/configure > index 4e28114d9c..59095a5866 100755 > --- a/configure > +++ b/configure > @@ -240,7 +240,7 @@ External library support: > --enable-libiec61883 enable iec61883 via libiec61883 [no] > --enable-libilbc enable iLBC de/encoding via libilbc [no] > --enable-libjack enable JACK audio sound server [no] > - --enable-libjxl enable JPEG XL decoding via libjxl [no] > + --enable-libjxl enable JPEG XL de/encoding via libjxl [no] > --enable-libklvanc enable Kernel Labs VANC processing [no] > --enable-libkvazaar enable HEVC encoding via libkvazaar [no] > --enable-liblensfun enable lensfun lens correction [no] > @@ -3332,6 +3332,7 @@ libgsm_ms_encoder_deps="libgsm" > libilbc_decoder_deps="libilbc" > libilbc_encoder_deps="libilbc" > libjxl_decoder_deps="libjxl libjxl_threads" > +libjxl_encoder_deps="libjxl libjxl_threads" > libkvazaar_encoder_deps="libkvazaar" > libmodplug_demuxer_deps="libmodplug" > libmp3lame_encoder_deps="libmp3lame" > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > index c00b0d3246..b208cc0097 100644 > --- a/libavcodec/Makefile > +++ b/libavcodec/Makefile > @@ -1061,6 +1061,7 @@ OBJS-$(CONFIG_LIBGSM_MS_ENCODER) += libgsmenc.o > OBJS-$(CONFIG_LIBILBC_DECODER) += libilbc.o > OBJS-$(CONFIG_LIBILBC_ENCODER) += libilbc.o > OBJS-$(CONFIG_LIBJXL_DECODER) += libjxldec.o libjxl.o > +OBJS-$(CONFIG_LIBJXL_ENCODER) += libjxlenc.o libjxl.o > OBJS-$(CONFIG_LIBKVAZAAR_ENCODER) += libkvazaar.o > OBJS-$(CONFIG_LIBMP3LAME_ENCODER) += libmp3lame.o > OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER) += libopencore-amr.o > diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c > index a9cd69dfce..db92fb7af5 100644 > --- a/libavcodec/allcodecs.c > +++ b/libavcodec/allcodecs.c > @@ -750,6 +750,7 @@ extern const FFCodec ff_libgsm_ms_decoder; > extern const FFCodec ff_libilbc_encoder; > extern const FFCodec ff_libilbc_decoder; > extern const FFCodec ff_libjxl_decoder; > +extern const FFCodec ff_libjxl_encoder; > extern const FFCodec ff_libmp3lame_encoder; > extern const FFCodec ff_libopencore_amrnb_encoder; > extern const FFCodec ff_libopencore_amrnb_decoder; > diff --git a/libavcodec/libjxlenc.c b/libavcodec/libjxlenc.c > new file mode 100644 > index 0000000000..802a543383 > --- /dev/null > +++ b/libavcodec/libjxlenc.c > @@ -0,0 +1,385 @@ > +/* > + * JPEG XL encoding support via libjxl > + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * JPEG XL encoder using libjxl > + */ > + > +#include "libavutil/avutil.h" > +#include "libavutil/error.h" > +#include "libavutil/frame.h" > +#include "libavutil/libm.h" > +#include "libavutil/opt.h" > +#include "libavutil/pixdesc.h" > +#include "libavutil/pixfmt.h" > +#include "libavutil/version.h" > + > +#include "avcodec.h" > +#include "codec_internal.h" > +#include "internal.h" I think this header is unnecessary. > + > +#include <jxl/encode.h> > +#include <jxl/thread_parallel_runner.h> > +#include "libjxl.h" > + > +typedef struct LibJxlEncodeContext { > + AVClass *class; > + void *runner; > + JxlEncoder *encoder; > + JxlEncoderFrameSettings *options; > + int effort; > + float distance; > + int modular; > + uint8_t *buffer; > + size_t buffer_size; > +} LibJxlEncodeContext; > + > +/** > + * Map a quality setting for -qscale roughly from libjpeg > + * quality numbers to libjxl's butteraugli distance for > + * photographic content. > + * > + * Setting distance explicitly is preferred, but this will > + * allow qscale to be used as a fallback. > + * > + * This function is continuous and injective on [0, 100] which > + * makes it monotonic. > + * > + * @param quality 0.0 to 100.0 quality setting, libjpeg quality > + * @return Butteraugli distance between 0.0 and 15.0 > + */ > +static float quality_to_distance(float quality) > +{ > + if (quality >= 100.0) > + return 0.0; > + else if (quality >= 90.0) > + return (100.0 - quality) * 0.10; > + else if (quality >= 30.0) > + return 0.1 + (100.0 - quality) * 0.09; > + else if (quality > 0.0) > + return 15.0 + (59.0 * quality - 4350.0) * quality / 9000.0; > + else > + return 15.0; > +} > + > +/** > + * Initalize the decoder on a per-frame basis. All of these need to be set > + * once each time the decoder is reset, which it must be each frame to make > + * the image2 muxer work. > + * > + * @return 0 upon success, negative on failure. > + */ > +static int libjxl_init_jxl_encoder(AVCodecContext *avctx) > +{ > + LibJxlEncodeContext *ctx = avctx->priv_data; > + > + /* reset the encoder every frame for image2 muxer */ > + JxlEncoderReset(ctx->encoder); > + > + ctx->options = JxlEncoderFrameSettingsCreate(ctx->encoder, NULL); > + if (!ctx->options) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoderOptions\n"); > + return AVERROR_EXTERNAL; > + } > + > + /* This needs to be set each time the decoder is reset */ > + if (JxlEncoderSetParallelRunner(ctx->encoder, JxlThreadParallelRunner, ctx->runner) > + != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n"); > + return AVERROR_EXTERNAL; > + } > + > + /* these shouldn't fail, libjxl bug notwithstanding */ > + if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_EFFORT, ctx->effort) > + != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set effort to: %d\n", ctx->effort); > + return AVERROR_EXTERNAL; > + } > + > + /* check for negative zero, our default */ > + if (1.0f / ctx->distance == 1.0f / -0.0f) { > + /* use ffmpeg.c -q option if passed */ > + if (avctx->flags & AV_CODEC_FLAG_QSCALE) > + ctx->distance = quality_to_distance((float)avctx->global_quality / FF_QP2LAMBDA); > + else > + /* default 1.0 matches cjxl */ > + ctx->distance = 1.0; > + } > + > + /* > + * 0.01 is the minimum distance accepted for lossy > + * interpreting any positive value less than this as minimum > + */ > + if (ctx->distance > 0.0 && ctx->distance < 0.01) > + ctx->distance = 0.01; > + if (JxlEncoderOptionsSetDistance(ctx->options, ctx->distance) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set distance: %f\n", ctx->distance); > + return AVERROR_EXTERNAL; > + } > + > + /* > + * In theory the library should automatically enable modular if necessary, > + * but it appears it won't at the moment due to a bug. This will still > + * work even if that is patched. > + */ > + if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_MODULAR, > + ctx->modular || ctx->distance <= 0.0 ? 1 : -1) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set modular\n"); > + return AVERROR_EXTERNAL; > + } > + > + return 0; > +} > + > +/** > + * Global encoder initialization. This only needs to be run once, > + * not every frame. > + */ > +static av_cold int libjxl_encode_init(AVCodecContext *avctx) > +{ > + LibJxlEncodeContext *ctx = avctx->priv_data; > + JxlMemoryManager manager; > + > + ff_libjxl_init_memory_manager(&manager); > + ctx->encoder = JxlEncoderCreate(&manager); > + if (!ctx->encoder) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoder\n"); > + return AVERROR_EXTERNAL; > + } > + > + ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count)); > + if (!ctx->runner) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n"); > + return AVERROR_EXTERNAL; > + } > + > + ctx->buffer_size = 4096; > + ctx->buffer = av_malloc(ctx->buffer_size); > + > + if (!ctx->buffer) { > + av_log(avctx, AV_LOG_ERROR, "Could not allocate encoding buffer\n"); > + return AVERROR(ENOMEM); > + } > + > + return 0; > +} > + > +/** > + * Encode an entire frame. Currently animation, is not supported by > + * this encoder, so this will always reinitialize a new still image > + * and encode a one-frame image (for image2 and image2pipe). > + */ > +static int libjxl_encode_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet) > +{ > + LibJxlEncodeContext *ctx = avctx->priv_data; > + AVFrameSideData *sd; > + const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(frame->format); > + JxlBasicInfo info; > + JxlColorEncoding jxl_color; > + JxlPixelFormat jxl_fmt; > + JxlEncoderStatus status; > + int ff_status; > + size_t available = ctx->buffer_size; > + size_t bytes_written = 0; > + uint8_t *next_out = ctx->buffer; > + > + ff_status = libjxl_init_jxl_encoder(avctx); > + if (ff_status) { > + av_log(avctx, AV_LOG_ERROR, "Error frame-initializing JxlEncoder\n"); > + return ff_status; > + } > + > + /* populate the basic info settings */ > + JxlEncoderInitBasicInfo(&info); > + jxl_fmt.num_channels = pix_desc->nb_components; > + info.xsize = frame->width; > + info.ysize = frame->height; > + info.num_extra_channels = (jxl_fmt.num_channels + 1) % 2; > + info.num_color_channels = jxl_fmt.num_channels - info.num_extra_channels; > + info.bits_per_sample = av_get_bits_per_pixel(pix_desc) / jxl_fmt.num_channels; > + info.alpha_bits = (info.num_extra_channels > 0) * info.bits_per_sample; > + if (pix_desc->flags & AV_PIX_FMT_FLAG_FLOAT) { > + info.exponent_bits_per_sample = info.bits_per_sample > 16 ? 8 : 5; > + info.alpha_exponent_bits = info.alpha_bits ? info.exponent_bits_per_sample : 0; > + jxl_fmt.data_type = info.bits_per_sample > 16 ? JXL_TYPE_FLOAT : JXL_TYPE_FLOAT16; > + JxlColorEncodingSetToLinearSRGB(&jxl_color, info.num_color_channels == 1); > + } else { > + info.exponent_bits_per_sample = 0; > + info.alpha_exponent_bits = 0; > + jxl_fmt.data_type = info.bits_per_sample <= 8 ? JXL_TYPE_UINT8 : JXL_TYPE_UINT16; > + JxlColorEncodingSetToSRGB(&jxl_color, info.num_color_channels == 1); > + } > + > + if (info.bits_per_sample > 16 > + || info.xsize > (1 << 18) || info.ysize > (1 << 18) > + || (info.xsize << 4) * (info.ysize << 4) > (1 << 20)) { > + /* > + * must upgrade codestream to level 10, from level 5 > + * the encoder will not do this automatically > + */ > + if (JxlEncoderSetCodestreamLevel(ctx->encoder, 10) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Could not upgrade JXL Codestream level.\n"); > + return AVERROR_EXTERNAL; > + } > + } > + > + /* bitexact lossless requires there to be no XYB transform */ > + info.uses_original_profile = ctx->distance <= 0.0; > + > + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_ICC_PROFILE); > + if (sd && sd->size && JxlEncoderSetICCProfile(ctx->encoder, sd->data, sd->size) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_WARNING, "Could not set ICC Profile\n"); > + } else if (info.uses_original_profile) { > + /* > + * the color encoding is not used if uses_original_profile is false > + * this just works around a bug in libjxl 0.7.0 and lower > + */ > + if (JxlEncoderSetColorEncoding(ctx->encoder, &jxl_color) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlColorEncoding\n"); > + return AVERROR_EXTERNAL; > + } > + } > + > + if (JxlEncoderSetBasicInfo(ctx->encoder, &info) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlBasicInfo\n"); > + return AVERROR_EXTERNAL; > + } > + > + jxl_fmt.endianness = JXL_NATIVE_ENDIAN; > + jxl_fmt.align = frame->linesize[0]; > + > + if (JxlEncoderAddImageFrame(ctx->options, &jxl_fmt, frame->data[0], jxl_fmt.align * info.ysize) != JXL_ENC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to add Image Frame: %d\n", status); > + return AVERROR_EXTERNAL; > + } > + > + /* > + * Run this after the last frame in the image has been passed. > + * TODO support animation > + */ > + JxlEncoderCloseInput(ctx->encoder); > + > + while (1) { > + status = JxlEncoderProcessOutput(ctx->encoder, &next_out, &available); > + if (status == JXL_ENC_ERROR) { > + av_log(avctx, AV_LOG_ERROR, "Unspecified libjxl error occurred\n"); > + return AVERROR_EXTERNAL; > + } > + bytes_written = ctx->buffer_size - available; > + /* all data passed has been encoded */ > + if (status == JXL_ENC_SUCCESS) > + break; > + if (status == JXL_ENC_NEED_MORE_OUTPUT) { > + /* > + * at the moment, libjxl has no way to > + * tell us how much space it actually needs > + * so we need to malloc loop > + */ > + uint8_t *temp; > + ctx->buffer_size = bytes_written * 2; Is it possible that available is still > 0 if JXL_ENC_NEED_MORE_OUTPUT is returned? In this case the above might even shrink buffer_size. > + temp = av_realloc(ctx->buffer, ctx->buffer_size); > + if (!temp) { > + av_freep(&ctx->buffer); If you free this, you will be in a scenario where ctx->buffer is NULL, yet ctx->buffer_size is > 0. This is inconsistent and might lead to crashs in JxlEncoderProcessOutput. So don't free this and only set buffer_size after the reallocation succeeded. > + av_log(avctx, AV_LOG_ERROR, "Error reallocating encoder buffer\n"); return AVERROR(ENOMEM) is enough. > + return AVERROR(ENOMEM); > + } > + ctx->buffer = temp; > + next_out = ctx->buffer + bytes_written; > + available = ctx->buffer_size - bytes_written; > + continue; > + } > + } > + /* > + * This buffer will be copied when the generic > + * code makes this packet refcounted, > + * so we can use the buffer again. > + */ > + pkt->data = ctx->buffer; > + pkt->size = bytes_written; > + *got_packet = 1; > + return 0; > +} > + > +static av_cold int libjxl_encode_close(AVCodecContext *avctx) > +{ > + LibJxlEncodeContext *ctx = avctx->priv_data; > + > + if (ctx->runner) > + JxlThreadParallelRunnerDestroy(ctx->runner); > + ctx->runner = NULL; > + > + /* > + * destroying the decoder also frees > + * ctx->options so we don't need to > + */ > + if (ctx->encoder) > + JxlEncoderDestroy(ctx->encoder); > + ctx->encoder = NULL; > + > + if (ctx->buffer) Unnecessary check. > + av_freep(&ctx->buffer); > + ctx->buffer = NULL; > + ctx->buffer_size = 0; Unnecessary: av_freep() resets ctx->buffer and resetting buffer_size is unnecessary. > + > + return 0; > +} > + > +#define OFFSET(x) offsetof(LibJxlEncodeContext, x) > +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM > + > +static const AVOption libjxl_encode_options[] = { > + { "effort", "Encoding effort", OFFSET(effort), AV_OPT_TYPE_INT, { .i64 = 7 }, 1, 9, VE }, > + { "distance", "Maximum Butteraugli distance (quality setting, " > + "lower = better, zero = lossless, default 1.0)", OFFSET(distance), AV_OPT_TYPE_FLOAT, { .dbl = -0.0 }, 0.0, 15.0, VE }, > + { "modular", "Force modular mode", OFFSET(modular), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE }, > + { NULL }, > +}; > + > +static const AVClass libjxl_encode_class = { > + .class_name = "libjxl", > + .item_name = av_default_item_name, > + .option = libjxl_encode_options, > + .version = LIBAVUTIL_VERSION_INT, > +}; > + > +const FFCodec ff_libjxl_encoder = { > + .p.name = "libjxl", > + .p.long_name = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"), > + .p.type = AVMEDIA_TYPE_VIDEO, > + .p.id = AV_CODEC_ID_JPEGXL, > + .priv_data_size = sizeof(LibJxlEncodeContext), > + .init = libjxl_encode_init, > + .encode2 = libjxl_encode_frame, > + .close = libjxl_encode_close, > + .p.capabilities = AV_CODEC_CAP_OTHER_THREADS, > + .caps_internal = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP, > + .p.pix_fmts = (const enum AVPixelFormat[]) { > + AV_PIX_FMT_RGB24, AV_PIX_FMT_RGBA, > + AV_PIX_FMT_RGB48, AV_PIX_FMT_RGBA64, > + AV_PIX_FMT_GRAY8, AV_PIX_FMT_YA8, > + AV_PIX_FMT_GRAY16, AV_PIX_FMT_YA16, > + AV_PIX_FMT_GRAYF32, > + AV_PIX_FMT_NONE > + }, > + .p.priv_class = &libjxl_encode_class, > + .p.wrapper_name = "libjxl", > +};
On 3/23/22 11:08, Andreas Rheinhardt wrote: >> + temp = av_realloc(ctx->buffer, ctx->buffer_size); >> + if (!temp) { >> + av_freep(&ctx->buffer); > If you free this, you will be in a scenario where ctx->buffer is NULL, > yet ctx->buffer_size is > 0. This is inconsistent and might lead to > crashs in JxlEncoderProcessOutput. So don't free this and only set > buffer_size after the reallocation succeeded. Does it matter what ctx->buffer_size is if ctx->buffer is NULL? On 3/23/22 11:08, Andreas Rheinhardt wrote: > Unnecessary: av_freep() resets ctx->buffer and resetting buffer_size is > unnecessary. Here, you claim that resetting buffer_size is unnecessary but above you insist it is necessary. I'm confused here. Leo Izen (thebombzen)
Leo Izen: > On 3/23/22 11:08, Andreas Rheinhardt wrote: >>> + temp = av_realloc(ctx->buffer, ctx->buffer_size); >>> + if (!temp) { >>> + av_freep(&ctx->buffer); >> If you free this, you will be in a scenario where ctx->buffer is NULL, >> yet ctx->buffer_size is > 0. This is inconsistent and might lead to >> crashs in JxlEncoderProcessOutput. So don't free this and only set >> buffer_size after the reallocation succeeded. > Does it matter what ctx->buffer_size is if ctx->buffer is NULL? > On 3/23/22 11:08, Andreas Rheinhardt wrote: >> Unnecessary: av_freep() resets ctx->buffer and resetting buffer_size is >> unnecessary. > > Here, you claim that resetting buffer_size is unnecessary but above you > insist it is necessary. I'm confused here. > buffer_size is not used after libjxl_encode_close() at all, as this is the codec's close callback; libjxl_encode_frame() meanwhile may be called again after freeing and if it reaches JxlEncoderProcessOutput(), the latter would be called with next_out == NULL and available != 0. JxlEncoderProcessOutput() might check for this (and error out) or it may just crash. - Andreas
On 3/30/22 07:12, Andreas Rheinhardt wrote: > buffer_size is not used after libjxl_encode_close() at all, as this is > the codec's close callback; libjxl_encode_frame() meanwhile may be > called again after freeing and if it reaches JxlEncoderProcessOutput(), > the latter would be called with next_out == NULL and available != 0. > JxlEncoderProcessOutput() might check for this (and error out) or it may > just crash. > Can encode2 really be called again if it returns with AVERROR(ENOMEM)? I was under the impression that this preventing further calls. Leo Izen (thebombzen)
Leo Izen: > > On 3/30/22 07:12, Andreas Rheinhardt wrote: >> buffer_size is not used after libjxl_encode_close() at all, as this is >> the codec's close callback; libjxl_encode_frame() meanwhile may be >> called again after freeing and if it reaches JxlEncoderProcessOutput(), >> the latter would be called with next_out == NULL and available != 0. >> JxlEncoderProcessOutput() might check for this (and error out) or it may >> just crash. >> > Can encode2 really be called again if it returns with AVERROR(ENOMEM)? I > was under the impression that this preventing further calls. > Why should this be so? (E.g. a user might close other processes in order to free up memory upon receiving AVERROR(ENOMEM) and try again.) - Andreas
diff --git a/configure b/configure index 4e28114d9c..59095a5866 100755 --- a/configure +++ b/configure @@ -240,7 +240,7 @@ External library support: --enable-libiec61883 enable iec61883 via libiec61883 [no] --enable-libilbc enable iLBC de/encoding via libilbc [no] --enable-libjack enable JACK audio sound server [no] - --enable-libjxl enable JPEG XL decoding via libjxl [no] + --enable-libjxl enable JPEG XL de/encoding via libjxl [no] --enable-libklvanc enable Kernel Labs VANC processing [no] --enable-libkvazaar enable HEVC encoding via libkvazaar [no] --enable-liblensfun enable lensfun lens correction [no] @@ -3332,6 +3332,7 @@ libgsm_ms_encoder_deps="libgsm" libilbc_decoder_deps="libilbc" libilbc_encoder_deps="libilbc" libjxl_decoder_deps="libjxl libjxl_threads" +libjxl_encoder_deps="libjxl libjxl_threads" libkvazaar_encoder_deps="libkvazaar" libmodplug_demuxer_deps="libmodplug" libmp3lame_encoder_deps="libmp3lame" diff --git a/libavcodec/Makefile b/libavcodec/Makefile index c00b0d3246..b208cc0097 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1061,6 +1061,7 @@ OBJS-$(CONFIG_LIBGSM_MS_ENCODER) += libgsmenc.o OBJS-$(CONFIG_LIBILBC_DECODER) += libilbc.o OBJS-$(CONFIG_LIBILBC_ENCODER) += libilbc.o OBJS-$(CONFIG_LIBJXL_DECODER) += libjxldec.o libjxl.o +OBJS-$(CONFIG_LIBJXL_ENCODER) += libjxlenc.o libjxl.o OBJS-$(CONFIG_LIBKVAZAAR_ENCODER) += libkvazaar.o OBJS-$(CONFIG_LIBMP3LAME_ENCODER) += libmp3lame.o OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER) += libopencore-amr.o diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c index a9cd69dfce..db92fb7af5 100644 --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -750,6 +750,7 @@ extern const FFCodec ff_libgsm_ms_decoder; extern const FFCodec ff_libilbc_encoder; extern const FFCodec ff_libilbc_decoder; extern const FFCodec ff_libjxl_decoder; +extern const FFCodec ff_libjxl_encoder; extern const FFCodec ff_libmp3lame_encoder; extern const FFCodec ff_libopencore_amrnb_encoder; extern const FFCodec ff_libopencore_amrnb_decoder; diff --git a/libavcodec/libjxlenc.c b/libavcodec/libjxlenc.c new file mode 100644 index 0000000000..802a543383 --- /dev/null +++ b/libavcodec/libjxlenc.c @@ -0,0 +1,385 @@ +/* + * JPEG XL encoding support via libjxl + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL encoder using libjxl + */ + +#include "libavutil/avutil.h" +#include "libavutil/error.h" +#include "libavutil/frame.h" +#include "libavutil/libm.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "libavutil/pixfmt.h" +#include "libavutil/version.h" + +#include "avcodec.h" +#include "codec_internal.h" +#include "internal.h" + +#include <jxl/encode.h> +#include <jxl/thread_parallel_runner.h> +#include "libjxl.h" + +typedef struct LibJxlEncodeContext { + AVClass *class; + void *runner; + JxlEncoder *encoder; + JxlEncoderFrameSettings *options; + int effort; + float distance; + int modular; + uint8_t *buffer; + size_t buffer_size; +} LibJxlEncodeContext; + +/** + * Map a quality setting for -qscale roughly from libjpeg + * quality numbers to libjxl's butteraugli distance for + * photographic content. + * + * Setting distance explicitly is preferred, but this will + * allow qscale to be used as a fallback. + * + * This function is continuous and injective on [0, 100] which + * makes it monotonic. + * + * @param quality 0.0 to 100.0 quality setting, libjpeg quality + * @return Butteraugli distance between 0.0 and 15.0 + */ +static float quality_to_distance(float quality) +{ + if (quality >= 100.0) + return 0.0; + else if (quality >= 90.0) + return (100.0 - quality) * 0.10; + else if (quality >= 30.0) + return 0.1 + (100.0 - quality) * 0.09; + else if (quality > 0.0) + return 15.0 + (59.0 * quality - 4350.0) * quality / 9000.0; + else + return 15.0; +} + +/** + * Initalize the decoder on a per-frame basis. All of these need to be set + * once each time the decoder is reset, which it must be each frame to make + * the image2 muxer work. + * + * @return 0 upon success, negative on failure. + */ +static int libjxl_init_jxl_encoder(AVCodecContext *avctx) +{ + LibJxlEncodeContext *ctx = avctx->priv_data; + + /* reset the encoder every frame for image2 muxer */ + JxlEncoderReset(ctx->encoder); + + ctx->options = JxlEncoderFrameSettingsCreate(ctx->encoder, NULL); + if (!ctx->options) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoderOptions\n"); + return AVERROR_EXTERNAL; + } + + /* This needs to be set each time the decoder is reset */ + if (JxlEncoderSetParallelRunner(ctx->encoder, JxlThreadParallelRunner, ctx->runner) + != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n"); + return AVERROR_EXTERNAL; + } + + /* these shouldn't fail, libjxl bug notwithstanding */ + if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_EFFORT, ctx->effort) + != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set effort to: %d\n", ctx->effort); + return AVERROR_EXTERNAL; + } + + /* check for negative zero, our default */ + if (1.0f / ctx->distance == 1.0f / -0.0f) { + /* use ffmpeg.c -q option if passed */ + if (avctx->flags & AV_CODEC_FLAG_QSCALE) + ctx->distance = quality_to_distance((float)avctx->global_quality / FF_QP2LAMBDA); + else + /* default 1.0 matches cjxl */ + ctx->distance = 1.0; + } + + /* + * 0.01 is the minimum distance accepted for lossy + * interpreting any positive value less than this as minimum + */ + if (ctx->distance > 0.0 && ctx->distance < 0.01) + ctx->distance = 0.01; + if (JxlEncoderOptionsSetDistance(ctx->options, ctx->distance) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set distance: %f\n", ctx->distance); + return AVERROR_EXTERNAL; + } + + /* + * In theory the library should automatically enable modular if necessary, + * but it appears it won't at the moment due to a bug. This will still + * work even if that is patched. + */ + if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_MODULAR, + ctx->modular || ctx->distance <= 0.0 ? 1 : -1) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set modular\n"); + return AVERROR_EXTERNAL; + } + + return 0; +} + +/** + * Global encoder initialization. This only needs to be run once, + * not every frame. + */ +static av_cold int libjxl_encode_init(AVCodecContext *avctx) +{ + LibJxlEncodeContext *ctx = avctx->priv_data; + JxlMemoryManager manager; + + ff_libjxl_init_memory_manager(&manager); + ctx->encoder = JxlEncoderCreate(&manager); + if (!ctx->encoder) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoder\n"); + return AVERROR_EXTERNAL; + } + + ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count)); + if (!ctx->runner) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n"); + return AVERROR_EXTERNAL; + } + + ctx->buffer_size = 4096; + ctx->buffer = av_malloc(ctx->buffer_size); + + if (!ctx->buffer) { + av_log(avctx, AV_LOG_ERROR, "Could not allocate encoding buffer\n"); + return AVERROR(ENOMEM); + } + + return 0; +} + +/** + * Encode an entire frame. Currently animation, is not supported by + * this encoder, so this will always reinitialize a new still image + * and encode a one-frame image (for image2 and image2pipe). + */ +static int libjxl_encode_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet) +{ + LibJxlEncodeContext *ctx = avctx->priv_data; + AVFrameSideData *sd; + const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(frame->format); + JxlBasicInfo info; + JxlColorEncoding jxl_color; + JxlPixelFormat jxl_fmt; + JxlEncoderStatus status; + int ff_status; + size_t available = ctx->buffer_size; + size_t bytes_written = 0; + uint8_t *next_out = ctx->buffer; + + ff_status = libjxl_init_jxl_encoder(avctx); + if (ff_status) { + av_log(avctx, AV_LOG_ERROR, "Error frame-initializing JxlEncoder\n"); + return ff_status; + } + + /* populate the basic info settings */ + JxlEncoderInitBasicInfo(&info); + jxl_fmt.num_channels = pix_desc->nb_components; + info.xsize = frame->width; + info.ysize = frame->height; + info.num_extra_channels = (jxl_fmt.num_channels + 1) % 2; + info.num_color_channels = jxl_fmt.num_channels - info.num_extra_channels; + info.bits_per_sample = av_get_bits_per_pixel(pix_desc) / jxl_fmt.num_channels; + info.alpha_bits = (info.num_extra_channels > 0) * info.bits_per_sample; + if (pix_desc->flags & AV_PIX_FMT_FLAG_FLOAT) { + info.exponent_bits_per_sample = info.bits_per_sample > 16 ? 8 : 5; + info.alpha_exponent_bits = info.alpha_bits ? info.exponent_bits_per_sample : 0; + jxl_fmt.data_type = info.bits_per_sample > 16 ? JXL_TYPE_FLOAT : JXL_TYPE_FLOAT16; + JxlColorEncodingSetToLinearSRGB(&jxl_color, info.num_color_channels == 1); + } else { + info.exponent_bits_per_sample = 0; + info.alpha_exponent_bits = 0; + jxl_fmt.data_type = info.bits_per_sample <= 8 ? JXL_TYPE_UINT8 : JXL_TYPE_UINT16; + JxlColorEncodingSetToSRGB(&jxl_color, info.num_color_channels == 1); + } + + if (info.bits_per_sample > 16 + || info.xsize > (1 << 18) || info.ysize > (1 << 18) + || (info.xsize << 4) * (info.ysize << 4) > (1 << 20)) { + /* + * must upgrade codestream to level 10, from level 5 + * the encoder will not do this automatically + */ + if (JxlEncoderSetCodestreamLevel(ctx->encoder, 10) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Could not upgrade JXL Codestream level.\n"); + return AVERROR_EXTERNAL; + } + } + + /* bitexact lossless requires there to be no XYB transform */ + info.uses_original_profile = ctx->distance <= 0.0; + + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_ICC_PROFILE); + if (sd && sd->size && JxlEncoderSetICCProfile(ctx->encoder, sd->data, sd->size) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_WARNING, "Could not set ICC Profile\n"); + } else if (info.uses_original_profile) { + /* + * the color encoding is not used if uses_original_profile is false + * this just works around a bug in libjxl 0.7.0 and lower + */ + if (JxlEncoderSetColorEncoding(ctx->encoder, &jxl_color) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlColorEncoding\n"); + return AVERROR_EXTERNAL; + } + } + + if (JxlEncoderSetBasicInfo(ctx->encoder, &info) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlBasicInfo\n"); + return AVERROR_EXTERNAL; + } + + jxl_fmt.endianness = JXL_NATIVE_ENDIAN; + jxl_fmt.align = frame->linesize[0]; + + if (JxlEncoderAddImageFrame(ctx->options, &jxl_fmt, frame->data[0], jxl_fmt.align * info.ysize) != JXL_ENC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to add Image Frame: %d\n", status); + return AVERROR_EXTERNAL; + } + + /* + * Run this after the last frame in the image has been passed. + * TODO support animation + */ + JxlEncoderCloseInput(ctx->encoder); + + while (1) { + status = JxlEncoderProcessOutput(ctx->encoder, &next_out, &available); + if (status == JXL_ENC_ERROR) { + av_log(avctx, AV_LOG_ERROR, "Unspecified libjxl error occurred\n"); + return AVERROR_EXTERNAL; + } + bytes_written = ctx->buffer_size - available; + /* all data passed has been encoded */ + if (status == JXL_ENC_SUCCESS) + break; + if (status == JXL_ENC_NEED_MORE_OUTPUT) { + /* + * at the moment, libjxl has no way to + * tell us how much space it actually needs + * so we need to malloc loop + */ + uint8_t *temp; + ctx->buffer_size = bytes_written * 2; + temp = av_realloc(ctx->buffer, ctx->buffer_size); + if (!temp) { + av_freep(&ctx->buffer); + av_log(avctx, AV_LOG_ERROR, "Error reallocating encoder buffer\n"); + return AVERROR(ENOMEM); + } + ctx->buffer = temp; + next_out = ctx->buffer + bytes_written; + available = ctx->buffer_size - bytes_written; + continue; + } + } + /* + * This buffer will be copied when the generic + * code makes this packet refcounted, + * so we can use the buffer again. + */ + pkt->data = ctx->buffer; + pkt->size = bytes_written; + *got_packet = 1; + return 0; +} + +static av_cold int libjxl_encode_close(AVCodecContext *avctx) +{ + LibJxlEncodeContext *ctx = avctx->priv_data; + + if (ctx->runner) + JxlThreadParallelRunnerDestroy(ctx->runner); + ctx->runner = NULL; + + /* + * destroying the decoder also frees + * ctx->options so we don't need to + */ + if (ctx->encoder) + JxlEncoderDestroy(ctx->encoder); + ctx->encoder = NULL; + + if (ctx->buffer) + av_freep(&ctx->buffer); + ctx->buffer = NULL; + ctx->buffer_size = 0; + + return 0; +} + +#define OFFSET(x) offsetof(LibJxlEncodeContext, x) +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM + +static const AVOption libjxl_encode_options[] = { + { "effort", "Encoding effort", OFFSET(effort), AV_OPT_TYPE_INT, { .i64 = 7 }, 1, 9, VE }, + { "distance", "Maximum Butteraugli distance (quality setting, " + "lower = better, zero = lossless, default 1.0)", OFFSET(distance), AV_OPT_TYPE_FLOAT, { .dbl = -0.0 }, 0.0, 15.0, VE }, + { "modular", "Force modular mode", OFFSET(modular), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE }, + { NULL }, +}; + +static const AVClass libjxl_encode_class = { + .class_name = "libjxl", + .item_name = av_default_item_name, + .option = libjxl_encode_options, + .version = LIBAVUTIL_VERSION_INT, +}; + +const FFCodec ff_libjxl_encoder = { + .p.name = "libjxl", + .p.long_name = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"), + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_JPEGXL, + .priv_data_size = sizeof(LibJxlEncodeContext), + .init = libjxl_encode_init, + .encode2 = libjxl_encode_frame, + .close = libjxl_encode_close, + .p.capabilities = AV_CODEC_CAP_OTHER_THREADS, + .caps_internal = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP, + .p.pix_fmts = (const enum AVPixelFormat[]) { + AV_PIX_FMT_RGB24, AV_PIX_FMT_RGBA, + AV_PIX_FMT_RGB48, AV_PIX_FMT_RGBA64, + AV_PIX_FMT_GRAY8, AV_PIX_FMT_YA8, + AV_PIX_FMT_GRAY16, AV_PIX_FMT_YA16, + AV_PIX_FMT_GRAYF32, + AV_PIX_FMT_NONE + }, + .p.priv_class = &libjxl_encode_class, + .p.wrapper_name = "libjxl", +};