diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c
index 55e3a31e95..139e825216 100644
--- a/libavcodec/mpeg12dec.c
+++ b/libavcodec/mpeg12dec.c
@@ -535,14 +535,14 @@ static int mpeg_decode_mb(MpegEncContext *s, int16_t block[12][64])
 
         if (s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
             for (i = 0; i < mb_block_count; i++)
-                if ((ret = mpeg2_decode_block_intra(s, *s->pblocks[i], i)) < 0)
+                if ((ret = mpeg2_decode_block_intra(s, s->block[i], i)) < 0)
                     return ret;
         } else {
             for (i = 0; i < 6; i++) {
                 ret = ff_mpeg1_decode_block_intra(&s->gb,
                                                   s->intra_matrix,
                                                   s->intra_scantable.permutated,
-                                                  s->last_dc, *s->pblocks[i],
+                                                  s->last_dc, s->block[i],
                                                   i, s->qscale);
                 if (ret < 0) {
                     av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n",
@@ -760,7 +760,7 @@ static int mpeg_decode_mb(MpegEncContext *s, int16_t block[12][64])
 
                 for (i = 0; i < mb_block_count; i++) {
                     if (cbp & (1 << 11)) {
-                        if ((ret = mpeg2_decode_block_non_intra(s, *s->pblocks[i], i)) < 0)
+                        if ((ret = mpeg2_decode_block_non_intra(s, s->block[i], i)) < 0)
                             return ret;
                     } else {
                         s->block_last_index[i] = -1;
@@ -770,7 +770,7 @@ static int mpeg_decode_mb(MpegEncContext *s, int16_t block[12][64])
             } else {
                 for (i = 0; i < 6; i++) {
                     if (cbp & 32) {
-                        if ((ret = mpeg1_decode_block_inter(s, *s->pblocks[i], i)) < 0)
+                        if ((ret = mpeg1_decode_block_inter(s, s->block[i], i)) < 0)
                             return ret;
                     } else {
                         s->block_last_index[i] = -1;
@@ -1279,6 +1279,7 @@ static int mpeg_field_start(Mpeg1Context *s1, const uint8_t *buf, int buf_size)
 {
     MpegEncContext *s = &s1->mpeg_enc_ctx;
     AVCodecContext *avctx = s->avctx;
+    int second_field = 0;
     int ret;
 
     if (!(avctx->flags2 & AV_CODEC_FLAG2_CHUNKS)) {
@@ -1362,6 +1363,7 @@ static int mpeg_field_start(Mpeg1Context *s1, const uint8_t *buf, int buf_size)
         if (HAVE_THREADS && (avctx->active_thread_type & FF_THREAD_FRAME))
             ff_thread_finish_setup(avctx);
     } else { // second field
+        second_field = 1;
         if (!s->cur_pic.ptr) {
             av_log(s->avctx, AV_LOG_ERROR, "first field missing\n");
             return AVERROR_INVALIDDATA;
@@ -1389,6 +1391,16 @@ static int mpeg_field_start(Mpeg1Context *s1, const uint8_t *buf, int buf_size)
     if (avctx->hwaccel) {
         if ((ret = FF_HW_CALL(avctx, start_frame, buf, buf_size)) < 0)
             return ret;
+    } else if (s->codec_tag == MKTAG('V', 'C', 'R', '2')) {
+        // Exchange UV
+        FFSWAP(uint8_t*,  s->cur_pic.data[1],     s->cur_pic.data[2]);
+        FFSWAP(ptrdiff_t, s->cur_pic.linesize[1], s->cur_pic.linesize[2]);
+        if (!second_field) {
+            FFSWAP(uint8_t*,  s->next_pic.data[1],     s->next_pic.data[2]);
+            FFSWAP(ptrdiff_t, s->next_pic.linesize[1], s->next_pic.linesize[2]);
+            FFSWAP(uint8_t*,  s->last_pic.data[1],     s->last_pic.data[2]);
+            FFSWAP(ptrdiff_t, s->last_pic.linesize[1], s->last_pic.linesize[2]);
+        }
     }
 
     return 0;
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 8a8ff2fbd9..c5ed4701d0 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -365,8 +365,6 @@ av_cold void ff_mpv_idct_init(MpegEncContext *s)
 
 static int init_duplicate_context(MpegEncContext *s)
 {
-    int i;
-
     if (s->encoding) {
         s->me.map = av_mallocz(2 * ME_MAP_SIZE * sizeof(*s->me.map));
         if (!s->me.map)
@@ -382,15 +380,6 @@ static int init_duplicate_context(MpegEncContext *s)
         return AVERROR(ENOMEM);
     s->block = s->blocks[0];
 
-    for (i = 0; i < 12; i++) {
-        s->pblocks[i] = &s->block[i];
-    }
-
-    if (s->avctx->codec_tag == AV_RL32("VCR2")) {
-        // exchange uv
-        FFSWAP(void *, s->pblocks[4], s->pblocks[5]);
-    }
-
     if (s->out_format == FMT_H263) {
         int mb_height = s->msmpeg4_version == 6 /* VC-1 like */ ?
                             FFALIGN(s->mb_height, 2) : s->mb_height;
@@ -484,18 +473,12 @@ static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src)
 int ff_update_duplicate_context(MpegEncContext *dst, const MpegEncContext *src)
 {
     MpegEncContext bak;
-    int i, ret;
+    int ret;
     // FIXME copy only needed parts
     backup_duplicate_context(&bak, dst);
     memcpy(dst, src, sizeof(MpegEncContext));
     backup_duplicate_context(dst, &bak);
-    for (i = 0; i < 12; i++) {
-        dst->pblocks[i] = &dst->block[i];
-    }
-    if (dst->avctx->codec_tag == AV_RL32("VCR2")) {
-        // exchange uv
-        FFSWAP(void *, dst->pblocks[4], dst->pblocks[5]);
-    }
+
     ret = ff_mpv_framesize_alloc(dst->avctx, &dst->sc, dst->linesize);
     if (ret < 0) {
         av_log(dst->avctx, AV_LOG_ERROR, "failed to allocate context "
@@ -682,7 +665,6 @@ static void clear_context(MpegEncContext *s)
     s->dct_error_sum = NULL;
     s->block = NULL;
     s->blocks = NULL;
-    memset(s->pblocks, 0, sizeof(s->pblocks));
     s->ac_val_base = NULL;
     s->ac_val[0] =
     s->ac_val[1] =
diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index 4339a145aa..5af74acf95 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -466,7 +466,6 @@ typedef struct MpegEncContext {
     int rtp_payload_size;
 
     uint8_t *ptr_lastgob;
-    int16_t (*pblocks[12])[64];
 
     int16_t (*block)[64]; ///< points to one of the following blocks
     int16_t (*blocks)[12][64]; // for HQ mode we need to keep the best block
