diff mbox

[FFmpeg-devel] avcodec/dnxhdenc: fix DNxHR 444 encoding crashes see https://trac.ffmpeg.org/ticket/6649

Message ID 20170912132615.10975-1-frederic.devernay@m4x.org
State Accepted
Commit eec67f25224a48047da57be18b610b11b0fd0bfe
Headers show

Commit Message

Frederic Devernay Sept. 12, 2017, 1:26 p.m. UTC 
From: Frédéric Devernay <frederic.devernay@m4x.org>

---
 libavcodec/dnxhdenc.c | 16 ++++++++--------
 libavcodec/dnxhdenc.h |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

Comments

Carl Eugen Hoyos Sept. 15, 2017, 11:48 a.m. UTC | #1 
Ping?

Carl Eugen
Michael Niedermayer Sept. 15, 2017, 10:03 p.m. UTC | #2 
On Fri, Sep 15, 2017 at 01:48:11PM +0200, Carl Eugen Hoyos wrote:
> Ping?

i think its ok but i thought paul would review/apply as he commented
in the ticket and the author stated in it "but this should be reviewed by Paul B Mahol"

[...]
diff mbox

Patch

diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c
index 5a0e6de6a5..0d80381a2d 100644
--- a/libavcodec/dnxhdenc.c
+++ b/libavcodec/dnxhdenc.c
@@ -749,14 +749,14 @@  void dnxhd_get_blocks(DNXHDEncContext *ctx, int mb_x, int mb_y)
         ptr_y = &ctx->edge_buf_y[0];
         ptr_u = &ctx->edge_buf_uv[0][0];
         ptr_v = &ctx->edge_buf_uv[1][0];
-    } else if (ctx->bit_depth == 10 && vdsp->emulated_edge_mc && ((mb_x << 3) + 8 > ctx->m.avctx->width ||
-                                                                  (mb_y << 3) + 8 > ctx->m.avctx->height)) {
-        int y_w = ctx->m.avctx->width  - (mb_x << 3);
-        int y_h = ctx->m.avctx->height - (mb_y << 3);
+    } else if (ctx->bit_depth == 10 && vdsp->emulated_edge_mc && ((mb_x << 4) + 16 > ctx->m.avctx->width ||
+                                                                  (mb_y << 4) + 16 > ctx->m.avctx->height)) {
+        int y_w = ctx->m.avctx->width  - (mb_x << 4);
+        int y_h = ctx->m.avctx->height - (mb_y << 4);
         int uv_w = ctx->is_444 ? y_w : (y_w + 1) / 2;
         int uv_h = y_h;
-        linesize = 16;
-        uvlinesize = 8 + 8 * ctx->is_444;
+        linesize = 32;
+        uvlinesize = 16 + 16 * ctx->is_444;
 
         vdsp->emulated_edge_mc(&ctx->edge_buf_y[0], ptr_y,
                                linesize, ctx->m.linesize,
@@ -771,8 +771,8 @@  void dnxhd_get_blocks(DNXHDEncContext *ctx, int mb_x, int mb_y)
                                uvlinesize / 2, 16,
                                0, 0, uv_w, uv_h);
 
-        dct_y_offset =  bw * linesize;
-        dct_uv_offset = bw * uvlinesize;
+        dct_y_offset =  bw * linesize / 2;
+        dct_uv_offset = bw * uvlinesize / 2;
         ptr_y = &ctx->edge_buf_y[0];
         ptr_u = &ctx->edge_buf_uv[0][0];
         ptr_v = &ctx->edge_buf_uv[1][0];
diff --git a/libavcodec/dnxhdenc.h b/libavcodec/dnxhdenc.h
index 9b43f6e9c6..26c3eec695 100644
--- a/libavcodec/dnxhdenc.h
+++ b/libavcodec/dnxhdenc.h
@@ -75,8 +75,8 @@  typedef struct DNXHDEncContext {
     int intra_quant_bias;
 
     DECLARE_ALIGNED(16, int16_t, blocks)[12][64];
-    DECLARE_ALIGNED(16, uint8_t, edge_buf_y)[256];
-    DECLARE_ALIGNED(16, uint8_t, edge_buf_uv)[2][256];
+    DECLARE_ALIGNED(16, uint8_t, edge_buf_y)[512]; // has to hold 16x16 uint16 when depth=10
+    DECLARE_ALIGNED(16, uint8_t, edge_buf_uv)[2][512]; // has to hold 16x16 uint16_t when depth=10
 
     int      (*qmatrix_c)     [64];
     int      (*qmatrix_l)     [64];