diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h
index 307964b10e..0d6b79fe5d 100644
--- a/libavcodec/mpegvideo.h
+++ b/libavcodec/mpegvideo.h
@@ -200,14 +200,7 @@ typedef struct MPVContext {
     H263DSPContext h263dsp;
     int f_code;                 ///< forward MV resolution
     int b_code;                 ///< backward MV resolution for B-frames (MPEG-4)
-    int16_t (*p_mv_table_base)[2];
-    int16_t (*b_forw_mv_table_base)[2];
-    int16_t (*b_back_mv_table_base)[2];
-    int16_t (*b_bidir_forw_mv_table_base)[2];
-    int16_t (*b_bidir_back_mv_table_base)[2];
-    int16_t (*b_direct_mv_table_base)[2];
     int16_t (*p_field_mv_table_base)[2];
-    int16_t (*b_field_mv_table_base)[2];
     int16_t (*p_mv_table)[2];            ///< MV table (1MV per MB) P-frame encoding
     int16_t (*b_forw_mv_table)[2];       ///< MV table (1MV per MB) forward mode B-frame encoding
     int16_t (*b_back_mv_table)[2];       ///< MV table (1MV per MB) backward mode B-frame encoding
@@ -471,9 +464,6 @@ typedef struct MPVContext {
 
     int vbv_ignore_qmax;
 
-    /* temp buffers for rate control */
-    float *cplx_tab, *bits_tab;
-
     /* flag to indicate a reinitialization is required, e.g. after
      * a frame size change */
     int context_reinit;
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 1739472352..8b2de4770b 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -823,26 +823,26 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
     /* Allocate MV tables; the MV and MB tables will be copied
      * to slice contexts by ff_update_duplicate_context().  */
     mv_table_size = (s->mb_height + 2) * s->mb_stride + 1;
-    if (!FF_ALLOCZ_TYPED_ARRAY(s->p_mv_table_base,            mv_table_size) ||
-        !FF_ALLOCZ_TYPED_ARRAY(s->b_forw_mv_table_base,       mv_table_size) ||
-        !FF_ALLOCZ_TYPED_ARRAY(s->b_back_mv_table_base,       mv_table_size) ||
-        !FF_ALLOCZ_TYPED_ARRAY(s->b_bidir_forw_mv_table_base, mv_table_size) ||
-        !FF_ALLOCZ_TYPED_ARRAY(s->b_bidir_back_mv_table_base, mv_table_size) ||
-        !FF_ALLOCZ_TYPED_ARRAY(s->b_direct_mv_table_base,     mv_table_size))
+    if (!FF_ALLOCZ_TYPED_ARRAY(m->p_mv_table_base,            mv_table_size) ||
+        !FF_ALLOCZ_TYPED_ARRAY(m->b_forw_mv_table_base,       mv_table_size) ||
+        !FF_ALLOCZ_TYPED_ARRAY(m->b_back_mv_table_base,       mv_table_size) ||
+        !FF_ALLOCZ_TYPED_ARRAY(m->b_bidir_forw_mv_table_base, mv_table_size) ||
+        !FF_ALLOCZ_TYPED_ARRAY(m->b_bidir_back_mv_table_base, mv_table_size) ||
+        !FF_ALLOCZ_TYPED_ARRAY(m->b_direct_mv_table_base,     mv_table_size))
         return AVERROR(ENOMEM);
-    s->p_mv_table            = s->p_mv_table_base + s->mb_stride + 1;
-    s->b_forw_mv_table       = s->b_forw_mv_table_base + s->mb_stride + 1;
-    s->b_back_mv_table       = s->b_back_mv_table_base + s->mb_stride + 1;
-    s->b_bidir_forw_mv_table = s->b_bidir_forw_mv_table_base + s->mb_stride + 1;
-    s->b_bidir_back_mv_table = s->b_bidir_back_mv_table_base + s->mb_stride + 1;
-    s->b_direct_mv_table     = s->b_direct_mv_table_base + s->mb_stride + 1;
+    s->p_mv_table            = m->p_mv_table_base + s->mb_stride + 1;
+    s->b_forw_mv_table       = m->b_forw_mv_table_base + s->mb_stride + 1;
+    s->b_back_mv_table       = m->b_back_mv_table_base + s->mb_stride + 1;
+    s->b_bidir_forw_mv_table = m->b_bidir_forw_mv_table_base + s->mb_stride + 1;
+    s->b_bidir_back_mv_table = m->b_bidir_back_mv_table_base + s->mb_stride + 1;
+    s->b_direct_mv_table     = m->b_direct_mv_table_base + s->mb_stride + 1;
 
     /* Allocate MB type table */
     mb_array_size = s->mb_height * s->mb_stride;
     if (!FF_ALLOCZ_TYPED_ARRAY(s->mb_type,      mb_array_size) ||
         !FF_ALLOCZ_TYPED_ARRAY(s->lambda_table, mb_array_size) ||
-        !FF_ALLOC_TYPED_ARRAY (s->cplx_tab,     mb_array_size) ||
-        !FF_ALLOC_TYPED_ARRAY (s->bits_tab,     mb_array_size))
+        !FF_ALLOC_TYPED_ARRAY (m->cplx_tab,     mb_array_size) ||
+        !FF_ALLOC_TYPED_ARRAY (m->bits_tab,     mb_array_size))
         return AVERROR(ENOMEM);
 
 #define ALLOCZ_ARRAYS(p, mult, numb) ((p) = av_calloc(numb, mult * sizeof(*(p))))
@@ -850,7 +850,7 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
         (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME)) {
         int16_t (*tmp1)[2];
         uint8_t *tmp2;
-        if (!(tmp1 = ALLOCZ_ARRAYS(s->b_field_mv_table_base, 8, mv_table_size)) ||
+        if (!(tmp1 = ALLOCZ_ARRAYS(m->b_field_mv_table_base, 8, mv_table_size)) ||
             !(tmp2 = ALLOCZ_ARRAYS(s->b_field_select_table[0][0], 2 * 4, mv_table_size)) ||
             !ALLOCZ_ARRAYS(s->p_field_select_table[0], 2 * 2, mv_table_size))
             return AVERROR(ENOMEM);
@@ -995,21 +995,21 @@ av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
 
     av_freep(&avctx->stats_out);
 
-    av_freep(&s->p_mv_table_base);
-    av_freep(&s->b_forw_mv_table_base);
-    av_freep(&s->b_back_mv_table_base);
-    av_freep(&s->b_bidir_forw_mv_table_base);
-    av_freep(&s->b_bidir_back_mv_table_base);
-    av_freep(&s->b_direct_mv_table_base);
-    av_freep(&s->b_field_mv_table_base);
+    av_freep(&m->p_mv_table_base);
+    av_freep(&m->b_forw_mv_table_base);
+    av_freep(&m->b_back_mv_table_base);
+    av_freep(&m->b_bidir_forw_mv_table_base);
+    av_freep(&m->b_bidir_back_mv_table_base);
+    av_freep(&m->b_direct_mv_table_base);
+    av_freep(&m->b_field_mv_table_base);
     av_freep(&s->b_field_select_table[0][0]);
     av_freep(&s->p_field_select_table[0]);
 
     av_freep(&s->mb_type);
     av_freep(&s->lambda_table);
 
-    av_freep(&s->cplx_tab);
-    av_freep(&s->bits_tab);
+    av_freep(&m->cplx_tab);
+    av_freep(&m->bits_tab);
 
     if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
     if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
diff --git a/libavcodec/mpegvideoenc.h b/libavcodec/mpegvideoenc.h
index 86c86e0ddf..e124555b0a 100644
--- a/libavcodec/mpegvideoenc.h
+++ b/libavcodec/mpegvideoenc.h
@@ -84,11 +84,22 @@ typedef struct MPVMainEncContext {
 
     char *rc_eq;
 
+    /* temp buffers for rate control */
+    float *cplx_tab, *bits_tab;
+
     /* temporary frames used by b_frame_strategy == 2 */
     AVFrame *tmp_frames[MPVENC_MAX_B_FRAMES + 2];
     int b_frame_strategy;
     int b_sensitivity;
     int brd_scale;
+
+    int16_t (*p_mv_table_base)[2];
+    int16_t (*b_forw_mv_table_base)[2];
+    int16_t (*b_back_mv_table_base)[2];
+    int16_t (*b_bidir_forw_mv_table_base)[2];
+    int16_t (*b_bidir_back_mv_table_base)[2];
+    int16_t (*b_direct_mv_table_base)[2];
+    int16_t (*b_field_mv_table_base)[2];
 } MPVMainEncContext;
 
 #define UNI_AC_ENC_INDEX(run,level) ((run)*128 + (level))
diff --git a/libavcodec/ratecontrol.c b/libavcodec/ratecontrol.c
index fa28e6d61d..bbaa9191a0 100644
--- a/libavcodec/ratecontrol.c
+++ b/libavcodec/ratecontrol.c
@@ -758,8 +758,8 @@ static void adaptive_quantization(MPVMainEncContext *m, double q)
     const float border_masking       = m->border_masking;
     float bits_sum                   = 0.0;
     float cplx_sum                   = 0.0;
-    float *cplx_tab                  = s->cplx_tab;
-    float *bits_tab                  = s->bits_tab;
+    float *cplx_tab                  = m->cplx_tab;
+    float *bits_tab                  = m->bits_tab;
     const int qmin                   = s->avctx->mb_lmin;
     const int qmax                   = s->avctx->mb_lmax;
     Picture *const pic               = &s->current_picture;
