Message ID | 20220323110325.5499-2-leo.izen@gmail.com |
---|---|
State | New |
Headers | show |
Series | [FFmpeg-devel,v9,1/5] avcodec/jpegxl: add Jpeg XL image codec and parser | expand |
Context | Check | Description |
---|---|---|
yinshiyou/make_loongarch64 | success | Make finished |
yinshiyou/make_fate_loongarch64 | success | Make fate finished |
andriy/make_x86 | success | Make finished |
andriy/make_fate_x86 | success | Make fate finished |
Leo Izen: > This commit adds decoding support to libavcodec > for Jpeg XL images via the external library libjxl. > --- > MAINTAINERS | 1 + > configure | 5 + > doc/general_contents.texi | 7 + > libavcodec/Makefile | 1 + > libavcodec/allcodecs.c | 1 + > libavcodec/libjxl.c | 70 +++++++++ > libavcodec/libjxl.h | 48 ++++++ > libavcodec/libjxldec.c | 302 ++++++++++++++++++++++++++++++++++++++ > 8 files changed, 435 insertions(+) > create mode 100644 libavcodec/libjxl.c > create mode 100644 libavcodec/libjxl.h > create mode 100644 libavcodec/libjxldec.c > > diff --git a/MAINTAINERS b/MAINTAINERS > index 2e0de9e224..875d25ca89 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -195,6 +195,7 @@ Codecs: > libcodec2.c Tomas Härdin > libdirac* David Conrad > libdavs2.c Huiwen Ren > + libjxl*.c, libjxl.h Leo Izen > libgsm.c Michel Bardiaux > libkvazaar.c Arttu Ylä-Outinen > libopenh264enc.c Martin Storsjo, Linjie Fu > diff --git a/configure b/configure > index a7953ffc16..4e28114d9c 100755 > --- a/configure > +++ b/configure > @@ -240,6 +240,7 @@ External library support: > --enable-libiec61883 enable iec61883 via libiec61883 [no] > --enable-libilbc enable iLBC de/encoding via libilbc [no] > --enable-libjack enable JACK audio sound server [no] > + --enable-libjxl enable JPEG XL decoding via libjxl [no] > --enable-libklvanc enable Kernel Labs VANC processing [no] > --enable-libkvazaar enable HEVC encoding via libkvazaar [no] > --enable-liblensfun enable lensfun lens correction [no] > @@ -1833,6 +1834,7 @@ EXTERNAL_LIBRARY_LIST=" > libiec61883 > libilbc > libjack > + libjxl > libklvanc > libkvazaar > libmodplug > @@ -3329,6 +3331,7 @@ libgsm_ms_decoder_deps="libgsm" > libgsm_ms_encoder_deps="libgsm" > libilbc_decoder_deps="libilbc" > libilbc_encoder_deps="libilbc" > +libjxl_decoder_deps="libjxl libjxl_threads" > libkvazaar_encoder_deps="libkvazaar" > libmodplug_demuxer_deps="libmodplug" > libmp3lame_encoder_deps="libmp3lame" > @@ -6541,6 +6544,8 @@ enabled libgsm && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do > check_lib libgsm "${gsm_hdr}" gsm_create -lgsm && break; > done || die "ERROR: libgsm not found"; } > enabled libilbc && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc $pthreads_extralibs > +enabled libjxl && require_pkg_config libjxl "libjxl >= 0.7.0" jxl/decode.h JxlDecoderVersion && > + require_pkg_config libjxl_threads "libjxl_threads >= 0.7.0" jxl/thread_parallel_runner.h JxlThreadParallelRunner > enabled libklvanc && require libklvanc libklvanc/vanc.h klvanc_context_create -lklvanc > enabled libkvazaar && require_pkg_config libkvazaar "kvazaar >= 0.8.1" kvazaar.h kvz_api_get > enabled liblensfun && require_pkg_config liblensfun lensfun lensfun.h lf_db_new > diff --git a/doc/general_contents.texi b/doc/general_contents.texi > index fcd9da1b34..a893347fbe 100644 > --- a/doc/general_contents.texi > +++ b/doc/general_contents.texi > @@ -171,6 +171,13 @@ Go to @url{https://github.com/TimothyGu/libilbc} and follow the instructions for > installing the library. Then pass @code{--enable-libilbc} to configure to > enable it. > > +@section libjxl > + > +JPEG XL is an image format intended to fully replace legacy JPEG for an extended > +period of life. See @url{https://jpegxl.info/} for more information, and see > +@url{https://github.com/libjxl/libjxl} for the library source. You can pass > +@code{--enable-libjxl} to configure in order enable the libjxl wrapper. > + > @section libvpx > > FFmpeg can make use of the libvpx library for VP8/VP9 decoding and encoding. > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > index 3723601b3d..c00b0d3246 100644 > --- a/libavcodec/Makefile > +++ b/libavcodec/Makefile > @@ -1060,6 +1060,7 @@ OBJS-$(CONFIG_LIBGSM_MS_DECODER) += libgsmdec.o > OBJS-$(CONFIG_LIBGSM_MS_ENCODER) += libgsmenc.o > OBJS-$(CONFIG_LIBILBC_DECODER) += libilbc.o > OBJS-$(CONFIG_LIBILBC_ENCODER) += libilbc.o > +OBJS-$(CONFIG_LIBJXL_DECODER) += libjxldec.o libjxl.o > OBJS-$(CONFIG_LIBKVAZAAR_ENCODER) += libkvazaar.o > OBJS-$(CONFIG_LIBMP3LAME_ENCODER) += libmp3lame.o > OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER) += libopencore-amr.o > diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c > index 22d56760ec..a9cd69dfce 100644 > --- a/libavcodec/allcodecs.c > +++ b/libavcodec/allcodecs.c > @@ -749,6 +749,7 @@ extern const FFCodec ff_libgsm_ms_encoder; > extern const FFCodec ff_libgsm_ms_decoder; > extern const FFCodec ff_libilbc_encoder; > extern const FFCodec ff_libilbc_decoder; > +extern const FFCodec ff_libjxl_decoder; > extern const FFCodec ff_libmp3lame_encoder; > extern const FFCodec ff_libopencore_amrnb_encoder; > extern const FFCodec ff_libopencore_amrnb_decoder; > diff --git a/libavcodec/libjxl.c b/libavcodec/libjxl.c > new file mode 100644 > index 0000000000..204d91d8a8 > --- /dev/null > +++ b/libavcodec/libjxl.c > @@ -0,0 +1,70 @@ > +/* > + * JPEG XL de/encoding via libjxl, common support implementation > + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * JPEG XL via libjxl common support implementation > + */ > + > +#include "libavutil/cpu.h" > +#include "libavutil/mem.h" > + > +#include <jxl/memory_manager.h> > +#include "libjxl.h" > + > +size_t ff_libjxl_get_threadcount(int threads) > +{ > + if (threads <= 0) > + return av_cpu_count(); > + if (threads == 1) > + return 0; > + return threads; > +} > + > +/** > + * Wrapper around av_malloc used as a jpegxl_alloc_func. > + * > + * @param opaque opaque pointer for jpegxl_alloc_func, always ignored > + * @param size Size in bytes for the memory block to be allocated > + * @return Pointer to the allocated block, or `NULL` if it cannot be allocated > + */ > +static void *libjxl_av_malloc(void *opaque, size_t size) > +{ > + return av_malloc(size); > +} > + > +/** > + * Wrapper around av_free used as a jpegxl_free_func. > + * > + * @param opaque opaque pointer for jpegxl_free_func, always ignored > + * @param address Pointer to the allocated block, to free. `NULL` permitted as a no-op. > + */ > +static void libjxl_av_free(void *opaque, void *address) > +{ > + av_free(address); > +} > + > +void ff_libjxl_init_memory_manager(JxlMemoryManager *manager) > +{ > + manager->opaque = NULL; > + manager->alloc = &libjxl_av_malloc; > + manager->free = &libjxl_av_free; > +} > diff --git a/libavcodec/libjxl.h b/libavcodec/libjxl.h > new file mode 100644 > index 0000000000..5387c438fd > --- /dev/null > +++ b/libavcodec/libjxl.h > @@ -0,0 +1,48 @@ > +/* > + * JPEG XL de/encoding via libjxl, common support header > + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * JPEG XL via libjxl common support header > + */ > + > +#ifndef AVCODEC_LIBJXL_H > +#define AVCODEC_LIBJXL_H > + > +#include <jxl/memory_manager.h> > + > +/** > + * Transform threadcount in ffmpeg to one used by libjxl. > + * > + * @param threads ffmpeg's threads AVOption > + * @return thread count for libjxl's parallel runner > + */ > +size_t ff_libjxl_get_threadcount(int threads); > + > +/** > + * Initialize and populate a JxlMemoryManager > + * with av_malloc() and av_free() so libjxl will use these > + * functions. > + * @param manager a pointer to a JxlMemoryManager struct > + */ > +void ff_libjxl_init_memory_manager(JxlMemoryManager *manager); > + > +#endif /* AVCODEC_LIBJXL_H */ > diff --git a/libavcodec/libjxldec.c b/libavcodec/libjxldec.c > new file mode 100644 > index 0000000000..ad22971fbb > --- /dev/null > +++ b/libavcodec/libjxldec.c > @@ -0,0 +1,302 @@ > +/* > + * JPEG XL decoding support via libjxl > + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * JPEG XL decoder using libjxl > + */ > + > +#include "libavutil/avassert.h" > +#include "libavutil/common.h" > +#include "libavutil/error.h" > +#include "libavutil/mem.h" > +#include "libavutil/opt.h" > +#include "libavutil/pixdesc.h" > +#include "libavutil/pixfmt.h" > +#include "libavutil/frame.h" > +#include "libavutil/version.h" What do you need version for? And opt.h? > + > +#include "avcodec.h" > +#include "codec_internal.h" > +#include "internal.h" > + > +#include <jxl/decode.h> > +#include <jxl/thread_parallel_runner.h> > +#include "libjxl.h" > + > +typedef struct LibJxlDecodeContext { > + void *runner; > + JxlDecoder *decoder; > + JxlBasicInfo basic_info; > + JxlPixelFormat jxl_pixfmt; > + JxlDecoderStatus events; > + uint8_t *iccp; > + size_t iccp_len; > +} LibJxlDecodeContext; > + > +static int libjxl_init_jxl_decoder(AVCodecContext *avctx) > +{ > + LibJxlDecodeContext *ctx = avctx->priv_data; > + > + ctx->events = JXL_DEC_BASIC_INFO | JXL_DEC_FULL_IMAGE | JXL_DEC_FRAME | JXL_DEC_COLOR_ENCODING; > + if (JxlDecoderSubscribeEvents(ctx->decoder, ctx->events) != JXL_DEC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Error subscribing to JXL events\n"); > + return AVERROR_EXTERNAL; > + } > + > + if (JxlDecoderSetParallelRunner(ctx->decoder, JxlThreadParallelRunner, ctx->runner) != JXL_DEC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n"); > + return AVERROR_EXTERNAL; > + } > + > + memset(&ctx->basic_info, 0, sizeof(JxlBasicInfo)); > + memset(&ctx->jxl_pixfmt, 0, sizeof(JxlPixelFormat)); > + return 0; > +} > + > +static av_cold int libjxl_decode_init(AVCodecContext *avctx) > +{ > + LibJxlDecodeContext *ctx = avctx->priv_data; > + JxlMemoryManager manager; > + > + ff_libjxl_init_memory_manager(&manager); > + ctx->decoder = JxlDecoderCreate(&manager); > + if (!ctx->decoder) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlDecoder\n"); > + return AVERROR_EXTERNAL; > + } > + > + ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count)); > + if (!ctx->runner) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n"); > + return AVERROR_EXTERNAL; > + } > + > + return libjxl_init_jxl_decoder(avctx); > +} > + > +static enum AVPixelFormat libjxl_get_pix_fmt(AVCodecContext *avctx, JxlBasicInfo *basic_info, JxlPixelFormat *format) > +{ > + format->endianness = JXL_NATIVE_ENDIAN; > + format->num_channels = basic_info->num_color_channels + (basic_info->alpha_bits > 0); > + /* av_malloc handles alignment already */ > + format->align = 1; > + /* Gray */ > + if (basic_info->num_color_channels == 1) { > + if (basic_info->bits_per_sample <= 8) { > + format->data_type = JXL_TYPE_UINT8; > + return basic_info->alpha_bits ? AV_PIX_FMT_YA8 : AV_PIX_FMT_GRAY8; > + } > + if (basic_info->exponent_bits_per_sample || basic_info->bits_per_sample > 16) { > + if (basic_info->alpha_bits) > + return AV_PIX_FMT_NONE; > + format->data_type = JXL_TYPE_FLOAT; > + return AV_PIX_FMT_GRAYF32; > + } > + format->data_type = JXL_TYPE_UINT16; > + return basic_info->alpha_bits ? AV_PIX_FMT_YA16 : AV_PIX_FMT_GRAY16; > + } > + /* rgb only */ > + /* libjxl only supports packed RGB and gray output at the moment */ > + if (basic_info->num_color_channels == 3) { > + if (basic_info->bits_per_sample <= 8) { > + format->data_type = JXL_TYPE_UINT8; > + return basic_info->alpha_bits ? AV_PIX_FMT_RGBA : AV_PIX_FMT_RGB24; > + } > + if (basic_info->bits_per_sample > 16) > + av_log(avctx, AV_LOG_WARNING, "Downsampling larger integer to 16-bit via libjxl\n"); > + if (basic_info->exponent_bits_per_sample) > + av_log(avctx, AV_LOG_WARNING, "Downsampling float to 16-bit integer via libjxl\n"); > + format->data_type = JXL_TYPE_UINT16; > + return basic_info->alpha_bits ? AV_PIX_FMT_RGBA64 : AV_PIX_FMT_RGB48; > + } > + return AV_PIX_FMT_NONE; > +} > + > +static void libjxl_row_fill(void *avframe, size_t x, size_t y, size_t num_pixels, const void *pixels) > +{ > + AVFrame *frame = avframe; > + int bytes = av_get_padded_bits_per_pixel(av_pix_fmt_desc_get(frame->format)) / 8; > + size_t offset = y * frame->linesize[0] + x * bytes; > + memcpy(frame->data[0] + offset, pixels, num_pixels * bytes); > +} > + > +static int libjxl_decode_frame(AVCodecContext *avctx, void *avframe, int *got_frame, AVPacket *avpkt) > +{ > + LibJxlDecodeContext *ctx = avctx->priv_data; > + uint8_t *buf = avpkt->data; > + size_t remaining = avpkt->size; > + AVFrame *frame = avframe; > + JxlDecoderStatus status; > + int ff_status; The typical name for this variable would be ret; JxlDecoderStatus could be could jret. > + *got_frame = 0; > + > + while (1) { > + /* > + * it only returns JXL_DEC_ERROR here if the input > + * was not released since the last time this was called > + * if this happens, it's a programmer error Is it guaranteed that this will never happen, e.g. when ff_get_buffer() or anything else fails? > + */ > + status = JxlDecoderSetInput(ctx->decoder, buf, remaining); > + av_assert0(status != JXL_DEC_ERROR); > + > + status = JxlDecoderProcessInput(ctx->decoder); > + /* > + * JxlDecoderReleaseInput returns the number > + * of bytes remaining to be read, rather than > + * the number of bytes that it did read > + */ > + remaining = JxlDecoderReleaseInput(ctx->decoder); > + buf = avpkt->data + avpkt->size - remaining; > + > + switch(status) { > + case JXL_DEC_ERROR: > + av_log(avctx, AV_LOG_ERROR, "Unknown libjxl decode error\n"); > + return AVERROR_EXTERNAL; > + case JXL_DEC_NEED_MORE_INPUT: > + if (remaining == 0) { > + av_log(avctx, AV_LOG_WARNING, "Unexpected end of JXL codestream\n"); > + return AVERROR(EAGAIN); > + } > + av_log(avctx, AV_LOG_DEBUG, "NEED_MORE_INPUT event emitted\n"); > + continue; > + case JXL_DEC_BASIC_INFO: > + av_log(avctx, AV_LOG_DEBUG, "BASIC_INFO event emitted\n"); > + if (JxlDecoderGetBasicInfo(ctx->decoder, &ctx->basic_info) != JXL_DEC_SUCCESS) { > + /* > + * this should never happen > + * if it does it is likely a libjxl decoder bug > + */ > + av_log(avctx, AV_LOG_ERROR, "Bad libjxl basic info event\n"); > + return AVERROR_EXTERNAL; > + } > + avctx->pix_fmt = libjxl_get_pix_fmt(avctx, &ctx->basic_info, &ctx->jxl_pixfmt); > + if (avctx->pix_fmt == AV_PIX_FMT_NONE) { > + av_log(avctx, AV_LOG_ERROR, "Bad libjxl pixel format\n"); > + return AVERROR_EXTERNAL; > + } > + ff_status = ff_set_dimensions(avctx, ctx->basic_info.xsize, ctx->basic_info.ysize); > + if (ff_status < 0) > + return ff_status; > + /* > + * We rewind the decoder and ask for everything again > + * This futureproofs the decoder since it will make > + * adding a parser or a dedicated demuxer much easier > + */ > + buf = avpkt->data; > + remaining = avpkt->size; > + JxlDecoderRewind(ctx->decoder); > + ctx->events &= ~JXL_DEC_BASIC_INFO; > + if (JxlDecoderSubscribeEvents(ctx->decoder, ctx->events) != JXL_DEC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Error subscribing to JXL events after rewind\n"); > + return AVERROR_EXTERNAL; > + } > + continue; > + case JXL_DEC_COLOR_ENCODING: > + av_log(avctx, AV_LOG_DEBUG, "COLOR_ENCODING event emitted\n"); > + status = JxlDecoderGetICCProfileSize(ctx->decoder, &ctx->jxl_pixfmt, JXL_COLOR_PROFILE_TARGET_ORIGINAL, &ctx->iccp_len); > + if (status == JXL_DEC_SUCCESS && ctx->iccp_len > 0) { > + if (ctx->iccp) > + av_freep(&ctx->iccp); > + ctx->iccp = av_malloc(ctx->iccp_len); Use av_buffer_alloc() here, as this will allow to avoid the memcpy when attaching the profile to the frame. > + if (!ctx->iccp) > + return AVERROR(ENOMEM); > + status = JxlDecoderGetColorAsICCProfile(ctx->decoder, &ctx->jxl_pixfmt, JXL_COLOR_PROFILE_TARGET_ORIGINAL, ctx->iccp, ctx->iccp_len); > + if (status != JXL_DEC_SUCCESS) > + av_freep(&ctx->iccp); > + } > + continue; > + case JXL_DEC_FRAME: > + case JXL_DEC_NEED_IMAGE_OUT_BUFFER: > + /* > + * We don't do this at basic info time > + * because it will happen again when we > + * rewind anyway > + */ > + av_log(avctx, AV_LOG_DEBUG, "%s event emitted\n", status == JXL_DEC_FRAME ? "FRAME" : "NEED_IMAGE_OUT_BUFFER"); > + ff_status = ff_get_buffer(avctx, frame, 0); > + if (ff_status < 0) > + return ff_status; > + if (JxlDecoderSetImageOutCallback(ctx->decoder, &ctx->jxl_pixfmt, &libjxl_row_fill, frame) != JXL_DEC_SUCCESS) { > + av_log(avctx, AV_LOG_ERROR, "Bad libjxl dec need image out buffer event\n"); > + return AVERROR_EXTERNAL; > + } > + continue; > + case JXL_DEC_FULL_IMAGE: > + /* full image is one frame, even if animated */ > + av_log(avctx, AV_LOG_DEBUG, "FULL_IMAGE event emitted\n"); > + *got_frame = 1; > + frame->pict_type = AV_PICTURE_TYPE_I; > + frame->key_frame = 1; > + if (ctx->iccp) { > + AVFrameSideData *sd = av_frame_new_side_data(frame, AV_FRAME_DATA_ICC_PROFILE, ctx->iccp_len); > + if (!sd) > + return AVERROR(ENOMEM); > + memcpy(sd->data, ctx->iccp, ctx->iccp_len); > + av_dict_set(&sd->metadata, "name", "", 0); The name is optional; so don't set this. > + } > + return avpkt->size - remaining; > + case JXL_DEC_SUCCESS: > + av_log(avctx, AV_LOG_DEBUG, "SUCCESS event emitted\n"); > + /* > + * The file has finished decoding > + * reset the decoder to let us > + * reuse it again for the next image > + */ > + JxlDecoderReset(ctx->decoder); > + libjxl_init_jxl_decoder(avctx); > + buf = avpkt->data; > + remaining = avpkt->size; > + continue; > + default: > + av_log(avctx, AV_LOG_ERROR, "Bad libjxl event: %d\n", status); > + return AVERROR_EXTERNAL; > + } > + } > +} > + > +static av_cold int libjxl_decode_close(AVCodecContext *avctx) > +{ > + LibJxlDecodeContext *ctx = avctx->priv_data; > + if (ctx->runner) > + JxlThreadParallelRunnerDestroy(ctx->runner); > + ctx->runner = NULL; > + if (ctx->decoder) > + JxlDecoderDestroy(ctx->decoder); > + ctx->decoder = NULL; > + if (ctx->iccp) Unnecessary check > + av_freep(&ctx->iccp); > + return 0; > +} > + > +const FFCodec ff_libjxl_decoder = { > + .p.name = "libjxl", > + .p.long_name = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"), > + .p.type = AVMEDIA_TYPE_VIDEO, > + .p.id = AV_CODEC_ID_JPEGXL, > + .priv_data_size = sizeof(LibJxlDecodeContext), > + .init = libjxl_decode_init, > + .decode = libjxl_decode_frame, > + .close = libjxl_decode_close, > + .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_OTHER_THREADS, > + .caps_internal = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP, > + .p.wrapper_name = "libjxl", > +};
diff --git a/MAINTAINERS b/MAINTAINERS index 2e0de9e224..875d25ca89 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -195,6 +195,7 @@ Codecs: libcodec2.c Tomas Härdin libdirac* David Conrad libdavs2.c Huiwen Ren + libjxl*.c, libjxl.h Leo Izen libgsm.c Michel Bardiaux libkvazaar.c Arttu Ylä-Outinen libopenh264enc.c Martin Storsjo, Linjie Fu diff --git a/configure b/configure index a7953ffc16..4e28114d9c 100755 --- a/configure +++ b/configure @@ -240,6 +240,7 @@ External library support: --enable-libiec61883 enable iec61883 via libiec61883 [no] --enable-libilbc enable iLBC de/encoding via libilbc [no] --enable-libjack enable JACK audio sound server [no] + --enable-libjxl enable JPEG XL decoding via libjxl [no] --enable-libklvanc enable Kernel Labs VANC processing [no] --enable-libkvazaar enable HEVC encoding via libkvazaar [no] --enable-liblensfun enable lensfun lens correction [no] @@ -1833,6 +1834,7 @@ EXTERNAL_LIBRARY_LIST=" libiec61883 libilbc libjack + libjxl libklvanc libkvazaar libmodplug @@ -3329,6 +3331,7 @@ libgsm_ms_decoder_deps="libgsm" libgsm_ms_encoder_deps="libgsm" libilbc_decoder_deps="libilbc" libilbc_encoder_deps="libilbc" +libjxl_decoder_deps="libjxl libjxl_threads" libkvazaar_encoder_deps="libkvazaar" libmodplug_demuxer_deps="libmodplug" libmp3lame_encoder_deps="libmp3lame" @@ -6541,6 +6544,8 @@ enabled libgsm && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do check_lib libgsm "${gsm_hdr}" gsm_create -lgsm && break; done || die "ERROR: libgsm not found"; } enabled libilbc && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc $pthreads_extralibs +enabled libjxl && require_pkg_config libjxl "libjxl >= 0.7.0" jxl/decode.h JxlDecoderVersion && + require_pkg_config libjxl_threads "libjxl_threads >= 0.7.0" jxl/thread_parallel_runner.h JxlThreadParallelRunner enabled libklvanc && require libklvanc libklvanc/vanc.h klvanc_context_create -lklvanc enabled libkvazaar && require_pkg_config libkvazaar "kvazaar >= 0.8.1" kvazaar.h kvz_api_get enabled liblensfun && require_pkg_config liblensfun lensfun lensfun.h lf_db_new diff --git a/doc/general_contents.texi b/doc/general_contents.texi index fcd9da1b34..a893347fbe 100644 --- a/doc/general_contents.texi +++ b/doc/general_contents.texi @@ -171,6 +171,13 @@ Go to @url{https://github.com/TimothyGu/libilbc} and follow the instructions for installing the library. Then pass @code{--enable-libilbc} to configure to enable it. +@section libjxl + +JPEG XL is an image format intended to fully replace legacy JPEG for an extended +period of life. See @url{https://jpegxl.info/} for more information, and see +@url{https://github.com/libjxl/libjxl} for the library source. You can pass +@code{--enable-libjxl} to configure in order enable the libjxl wrapper. + @section libvpx FFmpeg can make use of the libvpx library for VP8/VP9 decoding and encoding. diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 3723601b3d..c00b0d3246 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1060,6 +1060,7 @@ OBJS-$(CONFIG_LIBGSM_MS_DECODER) += libgsmdec.o OBJS-$(CONFIG_LIBGSM_MS_ENCODER) += libgsmenc.o OBJS-$(CONFIG_LIBILBC_DECODER) += libilbc.o OBJS-$(CONFIG_LIBILBC_ENCODER) += libilbc.o +OBJS-$(CONFIG_LIBJXL_DECODER) += libjxldec.o libjxl.o OBJS-$(CONFIG_LIBKVAZAAR_ENCODER) += libkvazaar.o OBJS-$(CONFIG_LIBMP3LAME_ENCODER) += libmp3lame.o OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER) += libopencore-amr.o diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c index 22d56760ec..a9cd69dfce 100644 --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -749,6 +749,7 @@ extern const FFCodec ff_libgsm_ms_encoder; extern const FFCodec ff_libgsm_ms_decoder; extern const FFCodec ff_libilbc_encoder; extern const FFCodec ff_libilbc_decoder; +extern const FFCodec ff_libjxl_decoder; extern const FFCodec ff_libmp3lame_encoder; extern const FFCodec ff_libopencore_amrnb_encoder; extern const FFCodec ff_libopencore_amrnb_decoder; diff --git a/libavcodec/libjxl.c b/libavcodec/libjxl.c new file mode 100644 index 0000000000..204d91d8a8 --- /dev/null +++ b/libavcodec/libjxl.c @@ -0,0 +1,70 @@ +/* + * JPEG XL de/encoding via libjxl, common support implementation + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL via libjxl common support implementation + */ + +#include "libavutil/cpu.h" +#include "libavutil/mem.h" + +#include <jxl/memory_manager.h> +#include "libjxl.h" + +size_t ff_libjxl_get_threadcount(int threads) +{ + if (threads <= 0) + return av_cpu_count(); + if (threads == 1) + return 0; + return threads; +} + +/** + * Wrapper around av_malloc used as a jpegxl_alloc_func. + * + * @param opaque opaque pointer for jpegxl_alloc_func, always ignored + * @param size Size in bytes for the memory block to be allocated + * @return Pointer to the allocated block, or `NULL` if it cannot be allocated + */ +static void *libjxl_av_malloc(void *opaque, size_t size) +{ + return av_malloc(size); +} + +/** + * Wrapper around av_free used as a jpegxl_free_func. + * + * @param opaque opaque pointer for jpegxl_free_func, always ignored + * @param address Pointer to the allocated block, to free. `NULL` permitted as a no-op. + */ +static void libjxl_av_free(void *opaque, void *address) +{ + av_free(address); +} + +void ff_libjxl_init_memory_manager(JxlMemoryManager *manager) +{ + manager->opaque = NULL; + manager->alloc = &libjxl_av_malloc; + manager->free = &libjxl_av_free; +} diff --git a/libavcodec/libjxl.h b/libavcodec/libjxl.h new file mode 100644 index 0000000000..5387c438fd --- /dev/null +++ b/libavcodec/libjxl.h @@ -0,0 +1,48 @@ +/* + * JPEG XL de/encoding via libjxl, common support header + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL via libjxl common support header + */ + +#ifndef AVCODEC_LIBJXL_H +#define AVCODEC_LIBJXL_H + +#include <jxl/memory_manager.h> + +/** + * Transform threadcount in ffmpeg to one used by libjxl. + * + * @param threads ffmpeg's threads AVOption + * @return thread count for libjxl's parallel runner + */ +size_t ff_libjxl_get_threadcount(int threads); + +/** + * Initialize and populate a JxlMemoryManager + * with av_malloc() and av_free() so libjxl will use these + * functions. + * @param manager a pointer to a JxlMemoryManager struct + */ +void ff_libjxl_init_memory_manager(JxlMemoryManager *manager); + +#endif /* AVCODEC_LIBJXL_H */ diff --git a/libavcodec/libjxldec.c b/libavcodec/libjxldec.c new file mode 100644 index 0000000000..ad22971fbb --- /dev/null +++ b/libavcodec/libjxldec.c @@ -0,0 +1,302 @@ +/* + * JPEG XL decoding support via libjxl + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL decoder using libjxl + */ + +#include "libavutil/avassert.h" +#include "libavutil/common.h" +#include "libavutil/error.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "libavutil/pixfmt.h" +#include "libavutil/frame.h" +#include "libavutil/version.h" + +#include "avcodec.h" +#include "codec_internal.h" +#include "internal.h" + +#include <jxl/decode.h> +#include <jxl/thread_parallel_runner.h> +#include "libjxl.h" + +typedef struct LibJxlDecodeContext { + void *runner; + JxlDecoder *decoder; + JxlBasicInfo basic_info; + JxlPixelFormat jxl_pixfmt; + JxlDecoderStatus events; + uint8_t *iccp; + size_t iccp_len; +} LibJxlDecodeContext; + +static int libjxl_init_jxl_decoder(AVCodecContext *avctx) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + + ctx->events = JXL_DEC_BASIC_INFO | JXL_DEC_FULL_IMAGE | JXL_DEC_FRAME | JXL_DEC_COLOR_ENCODING; + if (JxlDecoderSubscribeEvents(ctx->decoder, ctx->events) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Error subscribing to JXL events\n"); + return AVERROR_EXTERNAL; + } + + if (JxlDecoderSetParallelRunner(ctx->decoder, JxlThreadParallelRunner, ctx->runner) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n"); + return AVERROR_EXTERNAL; + } + + memset(&ctx->basic_info, 0, sizeof(JxlBasicInfo)); + memset(&ctx->jxl_pixfmt, 0, sizeof(JxlPixelFormat)); + return 0; +} + +static av_cold int libjxl_decode_init(AVCodecContext *avctx) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + JxlMemoryManager manager; + + ff_libjxl_init_memory_manager(&manager); + ctx->decoder = JxlDecoderCreate(&manager); + if (!ctx->decoder) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlDecoder\n"); + return AVERROR_EXTERNAL; + } + + ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count)); + if (!ctx->runner) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n"); + return AVERROR_EXTERNAL; + } + + return libjxl_init_jxl_decoder(avctx); +} + +static enum AVPixelFormat libjxl_get_pix_fmt(AVCodecContext *avctx, JxlBasicInfo *basic_info, JxlPixelFormat *format) +{ + format->endianness = JXL_NATIVE_ENDIAN; + format->num_channels = basic_info->num_color_channels + (basic_info->alpha_bits > 0); + /* av_malloc handles alignment already */ + format->align = 1; + /* Gray */ + if (basic_info->num_color_channels == 1) { + if (basic_info->bits_per_sample <= 8) { + format->data_type = JXL_TYPE_UINT8; + return basic_info->alpha_bits ? AV_PIX_FMT_YA8 : AV_PIX_FMT_GRAY8; + } + if (basic_info->exponent_bits_per_sample || basic_info->bits_per_sample > 16) { + if (basic_info->alpha_bits) + return AV_PIX_FMT_NONE; + format->data_type = JXL_TYPE_FLOAT; + return AV_PIX_FMT_GRAYF32; + } + format->data_type = JXL_TYPE_UINT16; + return basic_info->alpha_bits ? AV_PIX_FMT_YA16 : AV_PIX_FMT_GRAY16; + } + /* rgb only */ + /* libjxl only supports packed RGB and gray output at the moment */ + if (basic_info->num_color_channels == 3) { + if (basic_info->bits_per_sample <= 8) { + format->data_type = JXL_TYPE_UINT8; + return basic_info->alpha_bits ? AV_PIX_FMT_RGBA : AV_PIX_FMT_RGB24; + } + if (basic_info->bits_per_sample > 16) + av_log(avctx, AV_LOG_WARNING, "Downsampling larger integer to 16-bit via libjxl\n"); + if (basic_info->exponent_bits_per_sample) + av_log(avctx, AV_LOG_WARNING, "Downsampling float to 16-bit integer via libjxl\n"); + format->data_type = JXL_TYPE_UINT16; + return basic_info->alpha_bits ? AV_PIX_FMT_RGBA64 : AV_PIX_FMT_RGB48; + } + return AV_PIX_FMT_NONE; +} + +static void libjxl_row_fill(void *avframe, size_t x, size_t y, size_t num_pixels, const void *pixels) +{ + AVFrame *frame = avframe; + int bytes = av_get_padded_bits_per_pixel(av_pix_fmt_desc_get(frame->format)) / 8; + size_t offset = y * frame->linesize[0] + x * bytes; + memcpy(frame->data[0] + offset, pixels, num_pixels * bytes); +} + +static int libjxl_decode_frame(AVCodecContext *avctx, void *avframe, int *got_frame, AVPacket *avpkt) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + uint8_t *buf = avpkt->data; + size_t remaining = avpkt->size; + AVFrame *frame = avframe; + JxlDecoderStatus status; + int ff_status; + *got_frame = 0; + + while (1) { + /* + * it only returns JXL_DEC_ERROR here if the input + * was not released since the last time this was called + * if this happens, it's a programmer error + */ + status = JxlDecoderSetInput(ctx->decoder, buf, remaining); + av_assert0(status != JXL_DEC_ERROR); + + status = JxlDecoderProcessInput(ctx->decoder); + /* + * JxlDecoderReleaseInput returns the number + * of bytes remaining to be read, rather than + * the number of bytes that it did read + */ + remaining = JxlDecoderReleaseInput(ctx->decoder); + buf = avpkt->data + avpkt->size - remaining; + + switch(status) { + case JXL_DEC_ERROR: + av_log(avctx, AV_LOG_ERROR, "Unknown libjxl decode error\n"); + return AVERROR_EXTERNAL; + case JXL_DEC_NEED_MORE_INPUT: + if (remaining == 0) { + av_log(avctx, AV_LOG_WARNING, "Unexpected end of JXL codestream\n"); + return AVERROR(EAGAIN); + } + av_log(avctx, AV_LOG_DEBUG, "NEED_MORE_INPUT event emitted\n"); + continue; + case JXL_DEC_BASIC_INFO: + av_log(avctx, AV_LOG_DEBUG, "BASIC_INFO event emitted\n"); + if (JxlDecoderGetBasicInfo(ctx->decoder, &ctx->basic_info) != JXL_DEC_SUCCESS) { + /* + * this should never happen + * if it does it is likely a libjxl decoder bug + */ + av_log(avctx, AV_LOG_ERROR, "Bad libjxl basic info event\n"); + return AVERROR_EXTERNAL; + } + avctx->pix_fmt = libjxl_get_pix_fmt(avctx, &ctx->basic_info, &ctx->jxl_pixfmt); + if (avctx->pix_fmt == AV_PIX_FMT_NONE) { + av_log(avctx, AV_LOG_ERROR, "Bad libjxl pixel format\n"); + return AVERROR_EXTERNAL; + } + ff_status = ff_set_dimensions(avctx, ctx->basic_info.xsize, ctx->basic_info.ysize); + if (ff_status < 0) + return ff_status; + /* + * We rewind the decoder and ask for everything again + * This futureproofs the decoder since it will make + * adding a parser or a dedicated demuxer much easier + */ + buf = avpkt->data; + remaining = avpkt->size; + JxlDecoderRewind(ctx->decoder); + ctx->events &= ~JXL_DEC_BASIC_INFO; + if (JxlDecoderSubscribeEvents(ctx->decoder, ctx->events) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Error subscribing to JXL events after rewind\n"); + return AVERROR_EXTERNAL; + } + continue; + case JXL_DEC_COLOR_ENCODING: + av_log(avctx, AV_LOG_DEBUG, "COLOR_ENCODING event emitted\n"); + status = JxlDecoderGetICCProfileSize(ctx->decoder, &ctx->jxl_pixfmt, JXL_COLOR_PROFILE_TARGET_ORIGINAL, &ctx->iccp_len); + if (status == JXL_DEC_SUCCESS && ctx->iccp_len > 0) { + if (ctx->iccp) + av_freep(&ctx->iccp); + ctx->iccp = av_malloc(ctx->iccp_len); + if (!ctx->iccp) + return AVERROR(ENOMEM); + status = JxlDecoderGetColorAsICCProfile(ctx->decoder, &ctx->jxl_pixfmt, JXL_COLOR_PROFILE_TARGET_ORIGINAL, ctx->iccp, ctx->iccp_len); + if (status != JXL_DEC_SUCCESS) + av_freep(&ctx->iccp); + } + continue; + case JXL_DEC_FRAME: + case JXL_DEC_NEED_IMAGE_OUT_BUFFER: + /* + * We don't do this at basic info time + * because it will happen again when we + * rewind anyway + */ + av_log(avctx, AV_LOG_DEBUG, "%s event emitted\n", status == JXL_DEC_FRAME ? "FRAME" : "NEED_IMAGE_OUT_BUFFER"); + ff_status = ff_get_buffer(avctx, frame, 0); + if (ff_status < 0) + return ff_status; + if (JxlDecoderSetImageOutCallback(ctx->decoder, &ctx->jxl_pixfmt, &libjxl_row_fill, frame) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Bad libjxl dec need image out buffer event\n"); + return AVERROR_EXTERNAL; + } + continue; + case JXL_DEC_FULL_IMAGE: + /* full image is one frame, even if animated */ + av_log(avctx, AV_LOG_DEBUG, "FULL_IMAGE event emitted\n"); + *got_frame = 1; + frame->pict_type = AV_PICTURE_TYPE_I; + frame->key_frame = 1; + if (ctx->iccp) { + AVFrameSideData *sd = av_frame_new_side_data(frame, AV_FRAME_DATA_ICC_PROFILE, ctx->iccp_len); + if (!sd) + return AVERROR(ENOMEM); + memcpy(sd->data, ctx->iccp, ctx->iccp_len); + av_dict_set(&sd->metadata, "name", "", 0); + } + return avpkt->size - remaining; + case JXL_DEC_SUCCESS: + av_log(avctx, AV_LOG_DEBUG, "SUCCESS event emitted\n"); + /* + * The file has finished decoding + * reset the decoder to let us + * reuse it again for the next image + */ + JxlDecoderReset(ctx->decoder); + libjxl_init_jxl_decoder(avctx); + buf = avpkt->data; + remaining = avpkt->size; + continue; + default: + av_log(avctx, AV_LOG_ERROR, "Bad libjxl event: %d\n", status); + return AVERROR_EXTERNAL; + } + } +} + +static av_cold int libjxl_decode_close(AVCodecContext *avctx) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + if (ctx->runner) + JxlThreadParallelRunnerDestroy(ctx->runner); + ctx->runner = NULL; + if (ctx->decoder) + JxlDecoderDestroy(ctx->decoder); + ctx->decoder = NULL; + if (ctx->iccp) + av_freep(&ctx->iccp); + return 0; +} + +const FFCodec ff_libjxl_decoder = { + .p.name = "libjxl", + .p.long_name = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"), + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_JPEGXL, + .priv_data_size = sizeof(LibJxlDecodeContext), + .init = libjxl_decode_init, + .decode = libjxl_decode_frame, + .close = libjxl_decode_close, + .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_OTHER_THREADS, + .caps_internal = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP, + .p.wrapper_name = "libjxl", +};