diff mbox series

[FFmpeg-devel,25/33] avcodec/mpegvideo: Fix crash when using lowres with 10bit MPEG-4

Message ID AM7PR03MB666076C2FA37B576F0A1E2808F209@AM7PR03MB6660.eurprd03.prod.outlook.com
State Accepted
Commit d4d87f2ac56e13aaad68f2dd2efc69a977e6a3bc
Headers show
Series [FFmpeg-devel,01/21] avcodec/h263: Remove declaration for inexistent function | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_ppc success Make finished
andriy/make_fate_ppc success Make fate finished
andriy/make_aarch64_jetson success Make finished
andriy/make_fate_aarch64_jetson success Make fate finished

Commit Message

Andreas Rheinhardt Jan. 26, 2022, 9:34 p.m. UTC 
In this case the macroblocks written to are smaller, yet
the MPEG-4 Simple Studio Profile code for 10bit DPCM ignored this;
e.g. in case of lowres = 2 or = 3, the sample mpeg4_sstp_dpcm.m4v
from the FATE-suite reads beyond the end of the buffer.

This commit fixes this by taking lowres into account.
The DPCM macroblocks of the aforementioned sample look
as good as can be expected after this patch; yet the non-DPCM
coded macroblocks are simply corrupt.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/mpegvideo.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 47603c2991..40494fe115 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -1629,13 +1629,17 @@  void mpv_reconstruct_mb_internal(MpegEncContext *s, int16_t block[12][64],
                     uint16_t *dest_pcm[3] = {(uint16_t*)dest_y, (uint16_t*)dest_cb, (uint16_t*)dest_cr};
                     int linesize[3] = {dct_linesize, uvlinesize, uvlinesize};
                     for(i = 0; i < 3; i++) {
+                        const int16_t *src = (*s->dpcm_macroblock)[i];
                         int idx = 0;
                         int vsub = i ? s->chroma_y_shift : 0;
                         int hsub = i ? s->chroma_x_shift : 0;
-                        for(h = 0; h < (16 >> vsub); h++){
-                            for(w = 0; w < (16 >> hsub); w++)
-                                dest_pcm[i][w] = (*s->dpcm_macroblock)[i][idx++];
+                        int lowres = lowres_flag ? s->avctx->lowres : 0;
+                        int step = 1 << lowres;
+                        for (h = 0; h < (16 >> (vsub + lowres)); h++){
+                            for (w = 0, idx = 0; w < (16 >> (hsub + lowres)); w++, idx += step)
+                                dest_pcm[i][w] = src[idx];
                             dest_pcm[i] += linesize[i] / 2;
+                            src         += (16 >> hsub) * step;
                         }
                     }
                 } else {
@@ -1644,13 +1648,17 @@  void mpv_reconstruct_mb_internal(MpegEncContext *s, int16_t block[12][64],
                     int linesize[3] = {dct_linesize, uvlinesize, uvlinesize};
                     av_assert2(s->dpcm_direction == -1);
                     for(i = 0; i < 3; i++) {
+                        const int16_t *src = (*s->dpcm_macroblock)[i];
                         int idx = 0;
                         int vsub = i ? s->chroma_y_shift : 0;
                         int hsub = i ? s->chroma_x_shift : 0;
+                        int lowres = lowres_flag ? s->avctx->lowres : 0;
+                        int step = 1 << lowres;
                         dest_pcm[i] += (linesize[i] / 2) * ((16 >> vsub) - 1);
-                        for (h = (16 >> vsub) - 1; h >= 0; h--) {
-                            for (w = (16 >> hsub) - 1; w >= 0; w--)
-                                dest_pcm[i][w] = (*s->dpcm_macroblock)[i][idx++];
+                        for (h = (16 >> (vsub + lowres)) - 1; h >= 0; h--){
+                            for (w = (16 >> (hsub + lowres)) - 1, idx = 0; w >= 0; w--, idx += step)
+                                dest_pcm[i][w] = src[idx];
+                            src += step * (16 >> hsub);
                             dest_pcm[i] -= linesize[i] / 2;
                         }
                     }