diff mbox series

[FFmpeg-devel,v9,3/5] avcodec/libjxl: add Jpeg XL encoding via libjxl

Message ID 20220323110325.5499-3-leo.izen@gmail.com
State New
Headers show
Series [FFmpeg-devel,v9,1/5] avcodec/jpegxl: add Jpeg XL image codec and parser | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Leo Izen March 23, 2022, 11:03 a.m. UTC
This commit adds encoding support to libavcodec
for Jpeg XL images via the external library libjxl.
---
 configure              |   3 +-
 libavcodec/Makefile    |   1 +
 libavcodec/allcodecs.c |   1 +
 libavcodec/libjxlenc.c | 385 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 389 insertions(+), 1 deletion(-)
 create mode 100644 libavcodec/libjxlenc.c

Comments

Andreas Rheinhardt March 23, 2022, 3:08 p.m. UTC | #1
Leo Izen:
> This commit adds encoding support to libavcodec
> for Jpeg XL images via the external library libjxl.
> ---
>  configure              |   3 +-
>  libavcodec/Makefile    |   1 +
>  libavcodec/allcodecs.c |   1 +
>  libavcodec/libjxlenc.c | 385 +++++++++++++++++++++++++++++++++++++++++
>  4 files changed, 389 insertions(+), 1 deletion(-)
>  create mode 100644 libavcodec/libjxlenc.c
> 
> diff --git a/configure b/configure
> index 4e28114d9c..59095a5866 100755
> --- a/configure
> +++ b/configure
> @@ -240,7 +240,7 @@ External library support:
>    --enable-libiec61883     enable iec61883 via libiec61883 [no]
>    --enable-libilbc         enable iLBC de/encoding via libilbc [no]
>    --enable-libjack         enable JACK audio sound server [no]
> -  --enable-libjxl          enable JPEG XL decoding via libjxl [no]
> +  --enable-libjxl          enable JPEG XL de/encoding via libjxl [no]
>    --enable-libklvanc       enable Kernel Labs VANC processing [no]
>    --enable-libkvazaar      enable HEVC encoding via libkvazaar [no]
>    --enable-liblensfun      enable lensfun lens correction [no]
> @@ -3332,6 +3332,7 @@ libgsm_ms_encoder_deps="libgsm"
>  libilbc_decoder_deps="libilbc"
>  libilbc_encoder_deps="libilbc"
>  libjxl_decoder_deps="libjxl libjxl_threads"
> +libjxl_encoder_deps="libjxl libjxl_threads"
>  libkvazaar_encoder_deps="libkvazaar"
>  libmodplug_demuxer_deps="libmodplug"
>  libmp3lame_encoder_deps="libmp3lame"
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index c00b0d3246..b208cc0097 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -1061,6 +1061,7 @@ OBJS-$(CONFIG_LIBGSM_MS_ENCODER)          += libgsmenc.o
>  OBJS-$(CONFIG_LIBILBC_DECODER)            += libilbc.o
>  OBJS-$(CONFIG_LIBILBC_ENCODER)            += libilbc.o
>  OBJS-$(CONFIG_LIBJXL_DECODER)             += libjxldec.o libjxl.o
> +OBJS-$(CONFIG_LIBJXL_ENCODER)             += libjxlenc.o libjxl.o
>  OBJS-$(CONFIG_LIBKVAZAAR_ENCODER)         += libkvazaar.o
>  OBJS-$(CONFIG_LIBMP3LAME_ENCODER)         += libmp3lame.o
>  OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER)  += libopencore-amr.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index a9cd69dfce..db92fb7af5 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -750,6 +750,7 @@ extern const FFCodec ff_libgsm_ms_decoder;
>  extern const FFCodec ff_libilbc_encoder;
>  extern const FFCodec ff_libilbc_decoder;
>  extern const FFCodec ff_libjxl_decoder;
> +extern const FFCodec ff_libjxl_encoder;
>  extern const FFCodec ff_libmp3lame_encoder;
>  extern const FFCodec ff_libopencore_amrnb_encoder;
>  extern const FFCodec ff_libopencore_amrnb_decoder;
> diff --git a/libavcodec/libjxlenc.c b/libavcodec/libjxlenc.c
> new file mode 100644
> index 0000000000..802a543383
> --- /dev/null
> +++ b/libavcodec/libjxlenc.c
> @@ -0,0 +1,385 @@
> +/*
> + * JPEG XL encoding support via libjxl
> + * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * JPEG XL encoder using libjxl
> + */
> +
> +#include "libavutil/avutil.h"
> +#include "libavutil/error.h"
> +#include "libavutil/frame.h"
> +#include "libavutil/libm.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/pixfmt.h"
> +#include "libavutil/version.h"
> +
> +#include "avcodec.h"
> +#include "codec_internal.h"
> +#include "internal.h"

I think this header is unnecessary.

> +
> +#include <jxl/encode.h>
> +#include <jxl/thread_parallel_runner.h>
> +#include "libjxl.h"
> +
> +typedef struct LibJxlEncodeContext {
> +    AVClass *class;
> +    void *runner;
> +    JxlEncoder *encoder;
> +    JxlEncoderFrameSettings *options;
> +    int effort;
> +    float distance;
> +    int modular;
> +    uint8_t *buffer;
> +    size_t buffer_size;
> +} LibJxlEncodeContext;
> +
> +/**
> + * Map a quality setting for -qscale roughly from libjpeg
> + * quality numbers to libjxl's butteraugli distance for
> + * photographic content.
> + *
> + * Setting distance explicitly is preferred, but this will
> + * allow qscale to be used as a fallback.
> + *
> + * This function is continuous and injective on [0, 100] which
> + * makes it monotonic.
> + *
> + * @param  quality 0.0 to 100.0 quality setting, libjpeg quality
> + * @return         Butteraugli distance between 0.0 and 15.0
> + */
> +static float quality_to_distance(float quality)
> +{
> +    if (quality >= 100.0)
> +        return 0.0;
> +    else if (quality >= 90.0)
> +        return (100.0 - quality) * 0.10;
> +    else if (quality >= 30.0)
> +        return 0.1 + (100.0 - quality) * 0.09;
> +    else if (quality > 0.0)
> +        return 15.0 + (59.0 * quality - 4350.0) * quality / 9000.0;
> +    else
> +        return 15.0;
> +}
> +
> +/**
> + * Initalize the decoder on a per-frame basis. All of these need to be set
> + * once each time the decoder is reset, which it must be each frame to make
> + * the image2 muxer work.
> + *
> + * @return       0 upon success, negative on failure.
> + */
> +static int libjxl_init_jxl_encoder(AVCodecContext *avctx)
> +{
> +    LibJxlEncodeContext *ctx = avctx->priv_data;
> +
> +    /* reset the encoder every frame for image2 muxer */
> +    JxlEncoderReset(ctx->encoder);
> +
> +    ctx->options = JxlEncoderFrameSettingsCreate(ctx->encoder, NULL);
> +    if (!ctx->options) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoderOptions\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    /* This needs to be set each time the decoder is reset */
> +    if (JxlEncoderSetParallelRunner(ctx->encoder, JxlThreadParallelRunner, ctx->runner)
> +            != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    /* these shouldn't fail, libjxl bug notwithstanding */
> +    if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_EFFORT, ctx->effort)
> +            != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to set effort to: %d\n", ctx->effort);
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    /* check for negative zero, our default */
> +    if (1.0f / ctx->distance == 1.0f / -0.0f) {
> +        /* use ffmpeg.c -q option if passed */
> +        if (avctx->flags & AV_CODEC_FLAG_QSCALE)
> +            ctx->distance = quality_to_distance((float)avctx->global_quality / FF_QP2LAMBDA);
> +        else
> +            /* default 1.0 matches cjxl */
> +            ctx->distance = 1.0;
> +    }
> +
> +    /*
> +     * 0.01 is the minimum distance accepted for lossy
> +     * interpreting any positive value less than this as minimum
> +     */
> +    if (ctx->distance > 0.0 && ctx->distance < 0.01)
> +        ctx->distance = 0.01;
> +    if (JxlEncoderOptionsSetDistance(ctx->options, ctx->distance) != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to set distance: %f\n", ctx->distance);
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    /*
> +     * In theory the library should automatically enable modular if necessary,
> +     * but it appears it won't at the moment due to a bug. This will still
> +     * work even if that is patched.
> +     */
> +    if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_MODULAR,
> +            ctx->modular || ctx->distance <= 0.0 ? 1 : -1) != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to set modular\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    return 0;
> +}
> +
> +/**
> + * Global encoder initialization. This only needs to be run once,
> + * not every frame.
> + */
> +static av_cold int libjxl_encode_init(AVCodecContext *avctx)
> +{
> +    LibJxlEncodeContext *ctx = avctx->priv_data;
> +    JxlMemoryManager manager;
> +
> +    ff_libjxl_init_memory_manager(&manager);
> +    ctx->encoder = JxlEncoderCreate(&manager);
> +    if (!ctx->encoder) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoder\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count));
> +    if (!ctx->runner) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    ctx->buffer_size = 4096;
> +    ctx->buffer = av_malloc(ctx->buffer_size);
> +
> +    if (!ctx->buffer) {
> +        av_log(avctx, AV_LOG_ERROR, "Could not allocate encoding buffer\n");
> +        return AVERROR(ENOMEM);
> +    }
> +
> +    return 0;
> +}
> +
> +/**
> + * Encode an entire frame. Currently animation, is not supported by
> + * this encoder, so this will always reinitialize a new still image
> + * and encode a one-frame image (for image2 and image2pipe).
> + */
> +static int libjxl_encode_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet)
> +{
> +    LibJxlEncodeContext *ctx = avctx->priv_data;
> +    AVFrameSideData *sd;
> +    const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(frame->format);
> +    JxlBasicInfo info;
> +    JxlColorEncoding jxl_color;
> +    JxlPixelFormat jxl_fmt;
> +    JxlEncoderStatus status;
> +    int ff_status;
> +    size_t available = ctx->buffer_size;
> +    size_t bytes_written = 0;
> +    uint8_t *next_out = ctx->buffer;
> +
> +    ff_status = libjxl_init_jxl_encoder(avctx);
> +    if (ff_status) {
> +        av_log(avctx, AV_LOG_ERROR, "Error frame-initializing JxlEncoder\n");
> +        return ff_status;
> +    }
> +
> +    /* populate the basic info settings */
> +    JxlEncoderInitBasicInfo(&info);
> +    jxl_fmt.num_channels = pix_desc->nb_components;
> +    info.xsize = frame->width;
> +    info.ysize = frame->height;
> +    info.num_extra_channels = (jxl_fmt.num_channels + 1) % 2;
> +    info.num_color_channels = jxl_fmt.num_channels - info.num_extra_channels;
> +    info.bits_per_sample = av_get_bits_per_pixel(pix_desc) / jxl_fmt.num_channels;
> +    info.alpha_bits = (info.num_extra_channels > 0) * info.bits_per_sample;
> +    if (pix_desc->flags & AV_PIX_FMT_FLAG_FLOAT) {
> +        info.exponent_bits_per_sample = info.bits_per_sample > 16 ? 8 : 5;
> +        info.alpha_exponent_bits = info.alpha_bits ? info.exponent_bits_per_sample : 0;
> +        jxl_fmt.data_type = info.bits_per_sample > 16 ? JXL_TYPE_FLOAT : JXL_TYPE_FLOAT16;
> +        JxlColorEncodingSetToLinearSRGB(&jxl_color, info.num_color_channels == 1);
> +    } else {
> +        info.exponent_bits_per_sample = 0;
> +        info.alpha_exponent_bits = 0;
> +        jxl_fmt.data_type = info.bits_per_sample <= 8 ? JXL_TYPE_UINT8 : JXL_TYPE_UINT16;
> +        JxlColorEncodingSetToSRGB(&jxl_color, info.num_color_channels == 1);
> +    }
> +
> +    if (info.bits_per_sample > 16
> +        || info.xsize > (1 << 18) || info.ysize > (1 << 18)
> +        || (info.xsize << 4) * (info.ysize << 4) > (1 << 20)) {
> +        /*
> +         * must upgrade codestream to level 10, from level 5
> +         * the encoder will not do this automatically
> +         */
> +        if (JxlEncoderSetCodestreamLevel(ctx->encoder, 10) != JXL_ENC_SUCCESS) {
> +            av_log(avctx, AV_LOG_ERROR, "Could not upgrade JXL Codestream level.\n");
> +            return AVERROR_EXTERNAL;
> +        }
> +    }
> +
> +    /* bitexact lossless requires there to be no XYB transform */
> +    info.uses_original_profile = ctx->distance <= 0.0;
> +
> +    sd = av_frame_get_side_data(frame, AV_FRAME_DATA_ICC_PROFILE);
> +    if (sd && sd->size && JxlEncoderSetICCProfile(ctx->encoder, sd->data, sd->size) != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_WARNING, "Could not set ICC Profile\n");
> +    } else if (info.uses_original_profile) {
> +        /*
> +        * the color encoding is not used if uses_original_profile is false
> +        * this just works around a bug in libjxl 0.7.0 and lower
> +        */
> +        if (JxlEncoderSetColorEncoding(ctx->encoder, &jxl_color) != JXL_ENC_SUCCESS) {
> +            av_log(avctx, AV_LOG_ERROR, "Failed to set JxlColorEncoding\n");
> +            return AVERROR_EXTERNAL;
> +        }
> +    }
> +
> +    if (JxlEncoderSetBasicInfo(ctx->encoder, &info) != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to set JxlBasicInfo\n");
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    jxl_fmt.endianness = JXL_NATIVE_ENDIAN;
> +    jxl_fmt.align = frame->linesize[0];
> +
> +    if (JxlEncoderAddImageFrame(ctx->options, &jxl_fmt, frame->data[0], jxl_fmt.align * info.ysize) != JXL_ENC_SUCCESS) {
> +        av_log(avctx, AV_LOG_ERROR, "Failed to add Image Frame: %d\n", status);
> +        return AVERROR_EXTERNAL;
> +    }
> +
> +    /*
> +     * Run this after the last frame in the image has been passed.
> +     * TODO support animation
> +     */
> +    JxlEncoderCloseInput(ctx->encoder);
> +
> +    while (1) {
> +        status = JxlEncoderProcessOutput(ctx->encoder, &next_out, &available);
> +        if (status == JXL_ENC_ERROR) {
> +            av_log(avctx, AV_LOG_ERROR, "Unspecified libjxl error occurred\n");
> +            return AVERROR_EXTERNAL;
> +        }
> +        bytes_written = ctx->buffer_size - available;
> +        /* all data passed has been encoded */
> +        if (status == JXL_ENC_SUCCESS)
> +            break;
> +        if (status == JXL_ENC_NEED_MORE_OUTPUT) {
> +            /*
> +             * at the moment, libjxl has no way to
> +             * tell us how much space it actually needs
> +             * so we need to malloc loop
> +             */
> +            uint8_t *temp;
> +            ctx->buffer_size = bytes_written * 2;

Is it possible that available is still > 0 if JXL_ENC_NEED_MORE_OUTPUT
is returned? In this case the above might even shrink buffer_size.

> +            temp = av_realloc(ctx->buffer, ctx->buffer_size);
> +            if (!temp) {
> +                av_freep(&ctx->buffer);

If you free this, you will be in a scenario where ctx->buffer is NULL,
yet ctx->buffer_size is > 0. This is inconsistent and might lead to
crashs in JxlEncoderProcessOutput. So don't free this and only set
buffer_size after the reallocation succeeded.

> +                av_log(avctx, AV_LOG_ERROR, "Error reallocating encoder buffer\n");

return AVERROR(ENOMEM) is enough.

> +                return AVERROR(ENOMEM);
> +            }
> +            ctx->buffer = temp;
> +            next_out = ctx->buffer + bytes_written;
> +            available = ctx->buffer_size - bytes_written;
> +            continue;
> +        }
> +    }
> +    /*
> +     * This buffer will be copied when the generic
> +     * code makes this packet refcounted,
> +     * so we can use the buffer again.
> +     */
> +    pkt->data = ctx->buffer;
> +    pkt->size = bytes_written;
> +    *got_packet = 1;
> +    return 0;
> +}
> +
> +static av_cold int libjxl_encode_close(AVCodecContext *avctx)
> +{
> +    LibJxlEncodeContext *ctx = avctx->priv_data;
> +
> +    if (ctx->runner)
> +        JxlThreadParallelRunnerDestroy(ctx->runner);
> +    ctx->runner = NULL;
> +
> +    /*
> +     * destroying the decoder also frees
> +     * ctx->options so we don't need to
> +     */
> +    if (ctx->encoder)
> +        JxlEncoderDestroy(ctx->encoder);
> +    ctx->encoder = NULL;
> +
> +    if (ctx->buffer)

Unnecessary check.

> +        av_freep(&ctx->buffer);
> +    ctx->buffer = NULL;
> +    ctx->buffer_size = 0;

Unnecessary: av_freep() resets ctx->buffer and resetting buffer_size is
unnecessary.

> +
> +    return 0;
> +}
> +
> +#define OFFSET(x) offsetof(LibJxlEncodeContext, x)
> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
> +
> +static const AVOption libjxl_encode_options[] = {
> +    { "effort",        "Encoding effort",                                  OFFSET(effort),     AV_OPT_TYPE_INT,    { .i64 =    7 },    1,     9, VE },
> +    { "distance",      "Maximum Butteraugli distance (quality setting, "
> +                        "lower = better, zero = lossless, default 1.0)",   OFFSET(distance),   AV_OPT_TYPE_FLOAT,  { .dbl = -0.0 },  0.0,  15.0, VE },
> +    { "modular",       "Force modular mode",                               OFFSET(modular),    AV_OPT_TYPE_INT,    { .i64 =    0 },    0,     1, VE },
> +    { NULL },
> +};
> +
> +static const AVClass libjxl_encode_class = {
> +    .class_name = "libjxl",
> +    .item_name  = av_default_item_name,
> +    .option     = libjxl_encode_options,
> +    .version    = LIBAVUTIL_VERSION_INT,
> +};
> +
> +const FFCodec ff_libjxl_encoder = {
> +    .p.name           = "libjxl",
> +    .p.long_name      = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"),
> +    .p.type           = AVMEDIA_TYPE_VIDEO,
> +    .p.id             = AV_CODEC_ID_JPEGXL,
> +    .priv_data_size   = sizeof(LibJxlEncodeContext),
> +    .init             = libjxl_encode_init,
> +    .encode2          = libjxl_encode_frame,
> +    .close            = libjxl_encode_close,
> +    .p.capabilities   = AV_CODEC_CAP_OTHER_THREADS,
> +    .caps_internal    = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP,
> +    .p.pix_fmts       = (const enum AVPixelFormat[]) {
> +        AV_PIX_FMT_RGB24, AV_PIX_FMT_RGBA,
> +        AV_PIX_FMT_RGB48, AV_PIX_FMT_RGBA64,
> +        AV_PIX_FMT_GRAY8, AV_PIX_FMT_YA8,
> +        AV_PIX_FMT_GRAY16, AV_PIX_FMT_YA16,
> +        AV_PIX_FMT_GRAYF32,
> +        AV_PIX_FMT_NONE
> +    },
> +    .p.priv_class     = &libjxl_encode_class,
> +    .p.wrapper_name   = "libjxl",
> +};
Leo Izen March 28, 2022, 9:16 p.m. UTC | #2
On 3/23/22 11:08, Andreas Rheinhardt wrote:
>> +            temp = av_realloc(ctx->buffer, ctx->buffer_size);
>> +            if (!temp) {
>> +                av_freep(&ctx->buffer);
> If you free this, you will be in a scenario where ctx->buffer is NULL,
> yet ctx->buffer_size is > 0. This is inconsistent and might lead to
> crashs in JxlEncoderProcessOutput. So don't free this and only set
> buffer_size after the reallocation succeeded.
Does it matter what ctx->buffer_size is if ctx->buffer is NULL?
On 3/23/22 11:08, Andreas Rheinhardt wrote:
> Unnecessary: av_freep() resets ctx->buffer and resetting buffer_size is
> unnecessary.

Here, you claim that resetting buffer_size is unnecessary but above you 
insist it is necessary. I'm confused here.

Leo Izen (thebombzen)
Andreas Rheinhardt March 30, 2022, 11:12 a.m. UTC | #3
Leo Izen:
> On 3/23/22 11:08, Andreas Rheinhardt wrote:
>>> +            temp = av_realloc(ctx->buffer, ctx->buffer_size);
>>> +            if (!temp) {
>>> +                av_freep(&ctx->buffer);
>> If you free this, you will be in a scenario where ctx->buffer is NULL,
>> yet ctx->buffer_size is > 0. This is inconsistent and might lead to
>> crashs in JxlEncoderProcessOutput. So don't free this and only set
>> buffer_size after the reallocation succeeded.
> Does it matter what ctx->buffer_size is if ctx->buffer is NULL?
> On 3/23/22 11:08, Andreas Rheinhardt wrote:
>> Unnecessary: av_freep() resets ctx->buffer and resetting buffer_size is
>> unnecessary.
> 
> Here, you claim that resetting buffer_size is unnecessary but above you
> insist it is necessary. I'm confused here.
> 

buffer_size is not used after libjxl_encode_close() at all, as this is
the codec's close callback; libjxl_encode_frame() meanwhile may be
called again after freeing and if it reaches JxlEncoderProcessOutput(),
the latter would be called with next_out == NULL and available != 0.
JxlEncoderProcessOutput() might check for this (and error out) or it may
just crash.

- Andreas
Leo Izen March 30, 2022, 1:37 p.m. UTC | #4
On 3/30/22 07:12, Andreas Rheinhardt wrote:
> buffer_size is not used after libjxl_encode_close() at all, as this is
> the codec's close callback; libjxl_encode_frame() meanwhile may be
> called again after freeing and if it reaches JxlEncoderProcessOutput(),
> the latter would be called with next_out == NULL and available != 0.
> JxlEncoderProcessOutput() might check for this (and error out) or it may
> just crash.
>
Can encode2 really be called again if it returns with AVERROR(ENOMEM)? I 
was under the impression that this preventing further calls.

Leo Izen (thebombzen)
Andreas Rheinhardt March 30, 2022, 1:43 p.m. UTC | #5
Leo Izen:
> 
> On 3/30/22 07:12, Andreas Rheinhardt wrote:
>> buffer_size is not used after libjxl_encode_close() at all, as this is
>> the codec's close callback; libjxl_encode_frame() meanwhile may be
>> called again after freeing and if it reaches JxlEncoderProcessOutput(),
>> the latter would be called with next_out == NULL and available != 0.
>> JxlEncoderProcessOutput() might check for this (and error out) or it may
>> just crash.
>>
> Can encode2 really be called again if it returns with AVERROR(ENOMEM)? I
> was under the impression that this preventing further calls.
> 

Why should this be so? (E.g. a user might close other processes in order
to free up memory upon receiving AVERROR(ENOMEM) and try again.)

- Andreas
diff mbox series

Patch

diff --git a/configure b/configure
index 4e28114d9c..59095a5866 100755
--- a/configure
+++ b/configure
@@ -240,7 +240,7 @@  External library support:
   --enable-libiec61883     enable iec61883 via libiec61883 [no]
   --enable-libilbc         enable iLBC de/encoding via libilbc [no]
   --enable-libjack         enable JACK audio sound server [no]
-  --enable-libjxl          enable JPEG XL decoding via libjxl [no]
+  --enable-libjxl          enable JPEG XL de/encoding via libjxl [no]
   --enable-libklvanc       enable Kernel Labs VANC processing [no]
   --enable-libkvazaar      enable HEVC encoding via libkvazaar [no]
   --enable-liblensfun      enable lensfun lens correction [no]
@@ -3332,6 +3332,7 @@  libgsm_ms_encoder_deps="libgsm"
 libilbc_decoder_deps="libilbc"
 libilbc_encoder_deps="libilbc"
 libjxl_decoder_deps="libjxl libjxl_threads"
+libjxl_encoder_deps="libjxl libjxl_threads"
 libkvazaar_encoder_deps="libkvazaar"
 libmodplug_demuxer_deps="libmodplug"
 libmp3lame_encoder_deps="libmp3lame"
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index c00b0d3246..b208cc0097 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -1061,6 +1061,7 @@  OBJS-$(CONFIG_LIBGSM_MS_ENCODER)          += libgsmenc.o
 OBJS-$(CONFIG_LIBILBC_DECODER)            += libilbc.o
 OBJS-$(CONFIG_LIBILBC_ENCODER)            += libilbc.o
 OBJS-$(CONFIG_LIBJXL_DECODER)             += libjxldec.o libjxl.o
+OBJS-$(CONFIG_LIBJXL_ENCODER)             += libjxlenc.o libjxl.o
 OBJS-$(CONFIG_LIBKVAZAAR_ENCODER)         += libkvazaar.o
 OBJS-$(CONFIG_LIBMP3LAME_ENCODER)         += libmp3lame.o
 OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER)  += libopencore-amr.o
diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
index a9cd69dfce..db92fb7af5 100644
--- a/libavcodec/allcodecs.c
+++ b/libavcodec/allcodecs.c
@@ -750,6 +750,7 @@  extern const FFCodec ff_libgsm_ms_decoder;
 extern const FFCodec ff_libilbc_encoder;
 extern const FFCodec ff_libilbc_decoder;
 extern const FFCodec ff_libjxl_decoder;
+extern const FFCodec ff_libjxl_encoder;
 extern const FFCodec ff_libmp3lame_encoder;
 extern const FFCodec ff_libopencore_amrnb_encoder;
 extern const FFCodec ff_libopencore_amrnb_decoder;
diff --git a/libavcodec/libjxlenc.c b/libavcodec/libjxlenc.c
new file mode 100644
index 0000000000..802a543383
--- /dev/null
+++ b/libavcodec/libjxlenc.c
@@ -0,0 +1,385 @@ 
+/*
+ * JPEG XL encoding support via libjxl
+ * Copyright (c) 2021 Leo Izen <leo.izen@gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * JPEG XL encoder using libjxl
+ */
+
+#include "libavutil/avutil.h"
+#include "libavutil/error.h"
+#include "libavutil/frame.h"
+#include "libavutil/libm.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/pixfmt.h"
+#include "libavutil/version.h"
+
+#include "avcodec.h"
+#include "codec_internal.h"
+#include "internal.h"
+
+#include <jxl/encode.h>
+#include <jxl/thread_parallel_runner.h>
+#include "libjxl.h"
+
+typedef struct LibJxlEncodeContext {
+    AVClass *class;
+    void *runner;
+    JxlEncoder *encoder;
+    JxlEncoderFrameSettings *options;
+    int effort;
+    float distance;
+    int modular;
+    uint8_t *buffer;
+    size_t buffer_size;
+} LibJxlEncodeContext;
+
+/**
+ * Map a quality setting for -qscale roughly from libjpeg
+ * quality numbers to libjxl's butteraugli distance for
+ * photographic content.
+ *
+ * Setting distance explicitly is preferred, but this will
+ * allow qscale to be used as a fallback.
+ *
+ * This function is continuous and injective on [0, 100] which
+ * makes it monotonic.
+ *
+ * @param  quality 0.0 to 100.0 quality setting, libjpeg quality
+ * @return         Butteraugli distance between 0.0 and 15.0
+ */
+static float quality_to_distance(float quality)
+{
+    if (quality >= 100.0)
+        return 0.0;
+    else if (quality >= 90.0)
+        return (100.0 - quality) * 0.10;
+    else if (quality >= 30.0)
+        return 0.1 + (100.0 - quality) * 0.09;
+    else if (quality > 0.0)
+        return 15.0 + (59.0 * quality - 4350.0) * quality / 9000.0;
+    else
+        return 15.0;
+}
+
+/**
+ * Initalize the decoder on a per-frame basis. All of these need to be set
+ * once each time the decoder is reset, which it must be each frame to make
+ * the image2 muxer work.
+ *
+ * @return       0 upon success, negative on failure.
+ */
+static int libjxl_init_jxl_encoder(AVCodecContext *avctx)
+{
+    LibJxlEncodeContext *ctx = avctx->priv_data;
+
+    /* reset the encoder every frame for image2 muxer */
+    JxlEncoderReset(ctx->encoder);
+
+    ctx->options = JxlEncoderFrameSettingsCreate(ctx->encoder, NULL);
+    if (!ctx->options) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoderOptions\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    /* This needs to be set each time the decoder is reset */
+    if (JxlEncoderSetParallelRunner(ctx->encoder, JxlThreadParallelRunner, ctx->runner)
+            != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    /* these shouldn't fail, libjxl bug notwithstanding */
+    if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_EFFORT, ctx->effort)
+            != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to set effort to: %d\n", ctx->effort);
+        return AVERROR_EXTERNAL;
+    }
+
+    /* check for negative zero, our default */
+    if (1.0f / ctx->distance == 1.0f / -0.0f) {
+        /* use ffmpeg.c -q option if passed */
+        if (avctx->flags & AV_CODEC_FLAG_QSCALE)
+            ctx->distance = quality_to_distance((float)avctx->global_quality / FF_QP2LAMBDA);
+        else
+            /* default 1.0 matches cjxl */
+            ctx->distance = 1.0;
+    }
+
+    /*
+     * 0.01 is the minimum distance accepted for lossy
+     * interpreting any positive value less than this as minimum
+     */
+    if (ctx->distance > 0.0 && ctx->distance < 0.01)
+        ctx->distance = 0.01;
+    if (JxlEncoderOptionsSetDistance(ctx->options, ctx->distance) != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to set distance: %f\n", ctx->distance);
+        return AVERROR_EXTERNAL;
+    }
+
+    /*
+     * In theory the library should automatically enable modular if necessary,
+     * but it appears it won't at the moment due to a bug. This will still
+     * work even if that is patched.
+     */
+    if (JxlEncoderFrameSettingsSetOption(ctx->options, JXL_ENC_FRAME_SETTING_MODULAR,
+            ctx->modular || ctx->distance <= 0.0 ? 1 : -1) != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to set modular\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+/**
+ * Global encoder initialization. This only needs to be run once,
+ * not every frame.
+ */
+static av_cold int libjxl_encode_init(AVCodecContext *avctx)
+{
+    LibJxlEncodeContext *ctx = avctx->priv_data;
+    JxlMemoryManager manager;
+
+    ff_libjxl_init_memory_manager(&manager);
+    ctx->encoder = JxlEncoderCreate(&manager);
+    if (!ctx->encoder) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to create JxlEncoder\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count));
+    if (!ctx->runner) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    ctx->buffer_size = 4096;
+    ctx->buffer = av_malloc(ctx->buffer_size);
+
+    if (!ctx->buffer) {
+        av_log(avctx, AV_LOG_ERROR, "Could not allocate encoding buffer\n");
+        return AVERROR(ENOMEM);
+    }
+
+    return 0;
+}
+
+/**
+ * Encode an entire frame. Currently animation, is not supported by
+ * this encoder, so this will always reinitialize a new still image
+ * and encode a one-frame image (for image2 and image2pipe).
+ */
+static int libjxl_encode_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet)
+{
+    LibJxlEncodeContext *ctx = avctx->priv_data;
+    AVFrameSideData *sd;
+    const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(frame->format);
+    JxlBasicInfo info;
+    JxlColorEncoding jxl_color;
+    JxlPixelFormat jxl_fmt;
+    JxlEncoderStatus status;
+    int ff_status;
+    size_t available = ctx->buffer_size;
+    size_t bytes_written = 0;
+    uint8_t *next_out = ctx->buffer;
+
+    ff_status = libjxl_init_jxl_encoder(avctx);
+    if (ff_status) {
+        av_log(avctx, AV_LOG_ERROR, "Error frame-initializing JxlEncoder\n");
+        return ff_status;
+    }
+
+    /* populate the basic info settings */
+    JxlEncoderInitBasicInfo(&info);
+    jxl_fmt.num_channels = pix_desc->nb_components;
+    info.xsize = frame->width;
+    info.ysize = frame->height;
+    info.num_extra_channels = (jxl_fmt.num_channels + 1) % 2;
+    info.num_color_channels = jxl_fmt.num_channels - info.num_extra_channels;
+    info.bits_per_sample = av_get_bits_per_pixel(pix_desc) / jxl_fmt.num_channels;
+    info.alpha_bits = (info.num_extra_channels > 0) * info.bits_per_sample;
+    if (pix_desc->flags & AV_PIX_FMT_FLAG_FLOAT) {
+        info.exponent_bits_per_sample = info.bits_per_sample > 16 ? 8 : 5;
+        info.alpha_exponent_bits = info.alpha_bits ? info.exponent_bits_per_sample : 0;
+        jxl_fmt.data_type = info.bits_per_sample > 16 ? JXL_TYPE_FLOAT : JXL_TYPE_FLOAT16;
+        JxlColorEncodingSetToLinearSRGB(&jxl_color, info.num_color_channels == 1);
+    } else {
+        info.exponent_bits_per_sample = 0;
+        info.alpha_exponent_bits = 0;
+        jxl_fmt.data_type = info.bits_per_sample <= 8 ? JXL_TYPE_UINT8 : JXL_TYPE_UINT16;
+        JxlColorEncodingSetToSRGB(&jxl_color, info.num_color_channels == 1);
+    }
+
+    if (info.bits_per_sample > 16
+        || info.xsize > (1 << 18) || info.ysize > (1 << 18)
+        || (info.xsize << 4) * (info.ysize << 4) > (1 << 20)) {
+        /*
+         * must upgrade codestream to level 10, from level 5
+         * the encoder will not do this automatically
+         */
+        if (JxlEncoderSetCodestreamLevel(ctx->encoder, 10) != JXL_ENC_SUCCESS) {
+            av_log(avctx, AV_LOG_ERROR, "Could not upgrade JXL Codestream level.\n");
+            return AVERROR_EXTERNAL;
+        }
+    }
+
+    /* bitexact lossless requires there to be no XYB transform */
+    info.uses_original_profile = ctx->distance <= 0.0;
+
+    sd = av_frame_get_side_data(frame, AV_FRAME_DATA_ICC_PROFILE);
+    if (sd && sd->size && JxlEncoderSetICCProfile(ctx->encoder, sd->data, sd->size) != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_WARNING, "Could not set ICC Profile\n");
+    } else if (info.uses_original_profile) {
+        /*
+        * the color encoding is not used if uses_original_profile is false
+        * this just works around a bug in libjxl 0.7.0 and lower
+        */
+        if (JxlEncoderSetColorEncoding(ctx->encoder, &jxl_color) != JXL_ENC_SUCCESS) {
+            av_log(avctx, AV_LOG_ERROR, "Failed to set JxlColorEncoding\n");
+            return AVERROR_EXTERNAL;
+        }
+    }
+
+    if (JxlEncoderSetBasicInfo(ctx->encoder, &info) != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to set JxlBasicInfo\n");
+        return AVERROR_EXTERNAL;
+    }
+
+    jxl_fmt.endianness = JXL_NATIVE_ENDIAN;
+    jxl_fmt.align = frame->linesize[0];
+
+    if (JxlEncoderAddImageFrame(ctx->options, &jxl_fmt, frame->data[0], jxl_fmt.align * info.ysize) != JXL_ENC_SUCCESS) {
+        av_log(avctx, AV_LOG_ERROR, "Failed to add Image Frame: %d\n", status);
+        return AVERROR_EXTERNAL;
+    }
+
+    /*
+     * Run this after the last frame in the image has been passed.
+     * TODO support animation
+     */
+    JxlEncoderCloseInput(ctx->encoder);
+
+    while (1) {
+        status = JxlEncoderProcessOutput(ctx->encoder, &next_out, &available);
+        if (status == JXL_ENC_ERROR) {
+            av_log(avctx, AV_LOG_ERROR, "Unspecified libjxl error occurred\n");
+            return AVERROR_EXTERNAL;
+        }
+        bytes_written = ctx->buffer_size - available;
+        /* all data passed has been encoded */
+        if (status == JXL_ENC_SUCCESS)
+            break;
+        if (status == JXL_ENC_NEED_MORE_OUTPUT) {
+            /*
+             * at the moment, libjxl has no way to
+             * tell us how much space it actually needs
+             * so we need to malloc loop
+             */
+            uint8_t *temp;
+            ctx->buffer_size = bytes_written * 2;
+            temp = av_realloc(ctx->buffer, ctx->buffer_size);
+            if (!temp) {
+                av_freep(&ctx->buffer);
+                av_log(avctx, AV_LOG_ERROR, "Error reallocating encoder buffer\n");
+                return AVERROR(ENOMEM);
+            }
+            ctx->buffer = temp;
+            next_out = ctx->buffer + bytes_written;
+            available = ctx->buffer_size - bytes_written;
+            continue;
+        }
+    }
+    /*
+     * This buffer will be copied when the generic
+     * code makes this packet refcounted,
+     * so we can use the buffer again.
+     */
+    pkt->data = ctx->buffer;
+    pkt->size = bytes_written;
+    *got_packet = 1;
+    return 0;
+}
+
+static av_cold int libjxl_encode_close(AVCodecContext *avctx)
+{
+    LibJxlEncodeContext *ctx = avctx->priv_data;
+
+    if (ctx->runner)
+        JxlThreadParallelRunnerDestroy(ctx->runner);
+    ctx->runner = NULL;
+
+    /*
+     * destroying the decoder also frees
+     * ctx->options so we don't need to
+     */
+    if (ctx->encoder)
+        JxlEncoderDestroy(ctx->encoder);
+    ctx->encoder = NULL;
+
+    if (ctx->buffer)
+        av_freep(&ctx->buffer);
+    ctx->buffer = NULL;
+    ctx->buffer_size = 0;
+
+    return 0;
+}
+
+#define OFFSET(x) offsetof(LibJxlEncodeContext, x)
+#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
+
+static const AVOption libjxl_encode_options[] = {
+    { "effort",        "Encoding effort",                                  OFFSET(effort),     AV_OPT_TYPE_INT,    { .i64 =    7 },    1,     9, VE },
+    { "distance",      "Maximum Butteraugli distance (quality setting, "
+                        "lower = better, zero = lossless, default 1.0)",   OFFSET(distance),   AV_OPT_TYPE_FLOAT,  { .dbl = -0.0 },  0.0,  15.0, VE },
+    { "modular",       "Force modular mode",                               OFFSET(modular),    AV_OPT_TYPE_INT,    { .i64 =    0 },    0,     1, VE },
+    { NULL },
+};
+
+static const AVClass libjxl_encode_class = {
+    .class_name = "libjxl",
+    .item_name  = av_default_item_name,
+    .option     = libjxl_encode_options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+const FFCodec ff_libjxl_encoder = {
+    .p.name           = "libjxl",
+    .p.long_name      = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"),
+    .p.type           = AVMEDIA_TYPE_VIDEO,
+    .p.id             = AV_CODEC_ID_JPEGXL,
+    .priv_data_size   = sizeof(LibJxlEncodeContext),
+    .init             = libjxl_encode_init,
+    .encode2          = libjxl_encode_frame,
+    .close            = libjxl_encode_close,
+    .p.capabilities   = AV_CODEC_CAP_OTHER_THREADS,
+    .caps_internal    = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP,
+    .p.pix_fmts       = (const enum AVPixelFormat[]) {
+        AV_PIX_FMT_RGB24, AV_PIX_FMT_RGBA,
+        AV_PIX_FMT_RGB48, AV_PIX_FMT_RGBA64,
+        AV_PIX_FMT_GRAY8, AV_PIX_FMT_YA8,
+        AV_PIX_FMT_GRAY16, AV_PIX_FMT_YA16,
+        AV_PIX_FMT_GRAYF32,
+        AV_PIX_FMT_NONE
+    },
+    .p.priv_class     = &libjxl_encode_class,
+    .p.wrapper_name   = "libjxl",
+};