diff mbox

[FFmpeg-devel] avcodec/prores_ks: fixed luma quantize if q >= MAX_STORED_Q

Message ID 20181228203008.26892-1-alex@mogurenko.com
State New
Headers show

Commit Message

Alex Mogurenko Dec. 28, 2018, 8:30 p.m. UTC 
problem occurs in slice quant estimation and slice encoding.
if slice quant >= MAX_STORED_Q we dont use pre-calculated quant matrices but generate new:

qmat = ctx->custom_q;
qmat_chroma = ctx->custom_q;

 for (i = 0; i < 64; i++) {
            qmat[i] = ctx->quant_mat[i] * quant;
            qmat_chroma[i] = ctx->quant_chroma_mat[i] * quant;
}

as you see qmat and qmat_chroma both point to same ctx->custom_q

result it will contain chroma qunatizers as
qmat_chroma[i] = ctx->quant_chroma_mat[i] * quant;

last in the loop, after all we pass qmat/qmat_chroma to function where we
estimate encoded slice size or encode slice:
estimate_slice_plane / encode_slice_plane

custom_chroma_q  was added same way as custom_q e.g to encoder context I would prefer to have it on stack thought
---
 libavcodec/proresenc_kostya.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

Comments

Derek Buitenhuis Dec. 29, 2018, 4:34 p.m. UTC | #1 
On 28/12/2018 20:30, Alex Mogurenko wrote:
> ---
>  libavcodec/proresenc_kostya.c | 6 ++++--
>  1 file changed, 4 insertions(+), 2 deletions(-)

I think it's OK.

I'll push it in 24 hours if noone else objects.

- Derek
Derek Buitenhuis Dec. 30, 2018, 4:35 p.m. UTC | #2 
On 29/12/2018 16:34, Derek Buitenhuis wrote:
> I think it's OK.
> 
> I'll push it in 24 hours if noone else objects.

Pushed.

- Derek
diff mbox

Patch

diff --git a/libavcodec/proresenc_kostya.c b/libavcodec/proresenc_kostya.c
index 9a77d24fb6..e045a972f1 100644
--- a/libavcodec/proresenc_kostya.c
+++ b/libavcodec/proresenc_kostya.c
@@ -222,6 +222,7 @@  typedef struct ProresThreadData {
     DECLARE_ALIGNED(16, int16_t, blocks)[MAX_PLANES][64 * 4 * MAX_MBS_PER_SLICE];
     DECLARE_ALIGNED(16, uint16_t, emu_buf)[16 * 16];
     int16_t custom_q[64];
+    int16_t custom_chroma_q[64];
     struct TrellisNode *nodes;
 } ProresThreadData;
 
@@ -232,6 +233,7 @@  typedef struct ProresContext {
     int16_t quants[MAX_STORED_Q][64];
     int16_t quants_chroma[MAX_STORED_Q][64];
     int16_t custom_q[64];
+    int16_t custom_chroma_q[64];
     const uint8_t *quant_mat;
     const uint8_t *quant_chroma_mat;
     const uint8_t *scantable;
@@ -574,7 +576,7 @@  static int encode_slice(AVCodecContext *avctx, const AVFrame *pic,
         qmat_chroma = ctx->quants_chroma[quant];
     } else {
         qmat = ctx->custom_q;
-        qmat_chroma = ctx->custom_q;
+        qmat_chroma = ctx->custom_chroma_q;
         for (i = 0; i < 64; i++) {
             qmat[i] = ctx->quant_mat[i] * quant;
             qmat_chroma[i] = ctx->quant_chroma_mat[i] * quant;
@@ -902,7 +904,7 @@  static int find_slice_quant(AVCodecContext *avctx,
                 qmat_chroma = ctx->quants_chroma[q];
             } else {
                 qmat = td->custom_q;
-                qmat_chroma = td->custom_q;
+                qmat_chroma = td->custom_chroma_q;
                 for (i = 0; i < 64; i++) {
                     qmat[i] = ctx->quant_mat[i] * q;
                     qmat_chroma[i] = ctx->quant_chroma_mat[i] * q;