diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 32ca037526..ad6157f0c1 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -18,8 +18,6 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
-#include <stdint.h>
-
 #include "libavutil/avassert.h"
 #include "libavutil/common.h"
 #include "libavutil/mem.h"
@@ -34,13 +32,13 @@
 
 static void av_noinline free_picture_tables(Picture *pic)
 {
-    av_buffer_unref(&pic->mbskip_table_buf);
-    av_buffer_unref(&pic->qscale_table_buf);
-    av_buffer_unref(&pic->mb_type_buf);
+    ff_refstruct_unref(&pic->mbskip_table);
+    ff_refstruct_unref(&pic->qscale_table_base);
+    ff_refstruct_unref(&pic->mb_type_base);
 
     for (int i = 0; i < 2; i++) {
-        av_buffer_unref(&pic->motion_val_buf[i]);
-        av_buffer_unref(&pic->ref_index_buf[i]);
+        ff_refstruct_unref(&pic->motion_val_base[i]);
+        ff_refstruct_unref(&pic->ref_index[i]);
     }
 
     pic->mb_width  =
@@ -135,18 +133,18 @@ static int handle_pic_linesizes(AVCodecContext *avctx, Picture *pic,
 static int alloc_picture_tables(BufferPoolContext *pools, Picture *pic,
                                 int mb_height)
 {
-#define GET_BUFFER(name, idx_suffix) do { \
-    pic->name ## _buf idx_suffix = av_buffer_pool_get(pools->name ## _pool); \
-    if (!pic->name ## _buf idx_suffix) \
+#define GET_BUFFER(name, buf_suffix, idx_suffix) do { \
+    pic->name ## buf_suffix idx_suffix = ff_refstruct_pool_get(pools->name ## _pool); \
+    if (!pic->name ## buf_suffix idx_suffix) \
         return AVERROR(ENOMEM); \
 } while (0)
-    GET_BUFFER(mbskip_table,);
-    GET_BUFFER(qscale_table,);
-    GET_BUFFER(mb_type,);
+    GET_BUFFER(mbskip_table,,);
+    GET_BUFFER(qscale_table, _base,);
+    GET_BUFFER(mb_type, _base,);
     if (pools->motion_val_pool) {
         for (int i = 0; i < 2; i++) {
-            GET_BUFFER(motion_val, [i]);
-            GET_BUFFER(ref_index,  [i]);
+            GET_BUFFER(ref_index,, [i]);
+            GET_BUFFER(motion_val, _base, [i]);
         }
     }
 #undef GET_BUFFER
@@ -166,7 +164,7 @@ int ff_alloc_picture(AVCodecContext *avctx, Picture *pic, MotionEstContext *me,
                      ScratchpadContext *sc, BufferPoolContext *pools,
                      int mb_height, ptrdiff_t *linesize, ptrdiff_t *uvlinesize)
 {
-    int i, ret;
+    int ret;
 
     if (handle_pic_linesizes(avctx, pic, me, sc,
                              *linesize, *uvlinesize) < 0)
@@ -179,20 +177,12 @@ int ff_alloc_picture(AVCodecContext *avctx, Picture *pic, MotionEstContext *me,
     if (ret < 0)
         goto fail;
 
-    pic->mbskip_table = pic->mbskip_table_buf->data;
-    memset(pic->mbskip_table, 0, pic->mbskip_table_buf->size);
-    pic->qscale_table = pic->qscale_table_buf->data       + 2 * pic->mb_stride + 1;
-    pic->mb_type      = (uint32_t*)pic->mb_type_buf->data + 2 * pic->mb_stride + 1;
-
-    if (pic->motion_val_buf[0]) {
-        for (i = 0; i < 2; i++) {
-            pic->motion_val[i] = (int16_t (*)[2])pic->motion_val_buf[i]->data + 4;
-            pic->ref_index[i]  = pic->ref_index_buf[i]->data;
-            /* FIXME: The output of H.263 with OBMC depends upon
-             * the earlier content of the buffer; therefore we
-             * reset it here. */
-            memset(pic->motion_val_buf[i]->data, 0, pic->motion_val_buf[i]->size);
-        }
+    pic->qscale_table = pic->qscale_table_base + 2 * pic->mb_stride + 1;
+    pic->mb_type      = pic->mb_type_base      + 2 * pic->mb_stride + 1;
+
+    if (pic->motion_val_base[0]) {
+        for (int i = 0; i < 2; i++)
+            pic->motion_val[i] = pic->motion_val_base[i] + 4;
     }
 
     return 0;
@@ -224,36 +214,24 @@ void ff_mpeg_unref_picture(Picture *pic)
     pic->coded_picture_number   = 0;
 }
 
-static int update_picture_tables(Picture *dst, const Picture *src)
+static void update_picture_tables(Picture *dst, const Picture *src)
 {
-    int i, ret;
-
-    ret  = av_buffer_replace(&dst->mbskip_table_buf, src->mbskip_table_buf);
-    ret |= av_buffer_replace(&dst->qscale_table_buf, src->qscale_table_buf);
-    ret |= av_buffer_replace(&dst->mb_type_buf,      src->mb_type_buf);
-    for (i = 0; i < 2; i++) {
-        ret |= av_buffer_replace(&dst->motion_val_buf[i], src->motion_val_buf[i]);
-        ret |= av_buffer_replace(&dst->ref_index_buf[i],  src->ref_index_buf[i]);
-    }
-
-    if (ret < 0) {
-        free_picture_tables(dst);
-        return ret;
+    ff_refstruct_replace(&dst->mbskip_table, src->mbskip_table);
+    ff_refstruct_replace(&dst->qscale_table_base, src->qscale_table_base);
+    ff_refstruct_replace(&dst->mb_type_base,      src->mb_type_base);
+    for (int i = 0; i < 2; i++) {
+        ff_refstruct_replace(&dst->motion_val_base[i], src->motion_val_base[i]);
+        ff_refstruct_replace(&dst->ref_index[i],  src->ref_index[i]);
     }
 
-    dst->mbskip_table  = src->mbskip_table;
     dst->qscale_table  = src->qscale_table;
     dst->mb_type       = src->mb_type;
-    for (i = 0; i < 2; i++) {
+    for (int i = 0; i < 2; i++)
         dst->motion_val[i] = src->motion_val[i];
-        dst->ref_index[i]  = src->ref_index[i];
-    }
 
     dst->mb_width  = src->mb_width;
     dst->mb_height = src->mb_height;
     dst->mb_stride = src->mb_stride;
-
-    return 0;
 }
 
 int ff_mpeg_ref_picture(Picture *dst, Picture *src)
@@ -269,9 +247,7 @@ int ff_mpeg_ref_picture(Picture *dst, Picture *src)
     if (ret < 0)
         goto fail;
 
-    ret = update_picture_tables(dst, src);
-    if (ret < 0)
-        goto fail;
+    update_picture_tables(dst, src);
 
     ff_refstruct_replace(&dst->hwaccel_picture_private,
                           src->hwaccel_picture_private);
diff --git a/libavcodec/mpegpicture.h b/libavcodec/mpegpicture.h
index a0bfd8250f..363732910a 100644
--- a/libavcodec/mpegpicture.h
+++ b/libavcodec/mpegpicture.h
@@ -23,9 +23,6 @@
 
 #include <stdint.h>
 
-#include "libavutil/buffer.h"
-#include "libavutil/frame.h"
-
 #include "avcodec.h"
 #include "motion_est.h"
 #include "threadframe.h"
@@ -43,11 +40,11 @@ typedef struct ScratchpadContext {
 } ScratchpadContext;
 
 typedef struct BufferPoolContext {
-    AVBufferPool *mbskip_table_pool;
-    AVBufferPool *qscale_table_pool;
-    AVBufferPool *mb_type_pool;
-    AVBufferPool *motion_val_pool;
-    AVBufferPool *ref_index_pool;
+    struct FFRefStructPool *mbskip_table_pool;
+    struct FFRefStructPool *qscale_table_pool;
+    struct FFRefStructPool *mb_type_pool;
+    struct FFRefStructPool *motion_val_pool;
+    struct FFRefStructPool *ref_index_pool;
     int alloc_mb_width;                         ///< mb_width  used to allocate tables
     int alloc_mb_height;                        ///< mb_height used to allocate tables
     int alloc_mb_stride;                        ///< mb_stride used to allocate tables
@@ -60,19 +57,17 @@ typedef struct Picture {
     struct AVFrame *f;
     ThreadFrame tf;
 
-    AVBufferRef *qscale_table_buf;
+    int8_t *qscale_table_base;
     int8_t *qscale_table;
 
-    AVBufferRef *motion_val_buf[2];
+    int16_t (*motion_val_base[2])[2];
     int16_t (*motion_val[2])[2];
 
-    AVBufferRef *mb_type_buf;
+    uint32_t *mb_type_base;
     uint32_t *mb_type;          ///< types and macros are defined in mpegutils.h
 
-    AVBufferRef *mbskip_table_buf;
     uint8_t *mbskip_table;
 
-    AVBufferRef *ref_index_buf[2];
     int8_t *ref_index[2];
 
     /// RefStruct reference for hardware accelerator private data
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 5728f4cee3..eab4451e1e 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -41,6 +41,7 @@
 #include "mpegutils.h"
 #include "mpegvideo.h"
 #include "mpegvideodata.h"
+#include "refstruct.h"
 
 static void dct_unquantize_mpeg1_intra_c(MpegEncContext *s,
                                    int16_t *block, int n, int qscale)
@@ -536,11 +537,11 @@ void ff_mpv_common_defaults(MpegEncContext *s)
 
 static void free_buffer_pools(BufferPoolContext *pools)
 {
-    av_buffer_pool_uninit(&pools->mbskip_table_pool);
-    av_buffer_pool_uninit(&pools->qscale_table_pool);
-    av_buffer_pool_uninit(&pools->mb_type_pool);
-    av_buffer_pool_uninit(&pools->motion_val_pool);
-    av_buffer_pool_uninit(&pools->ref_index_pool);
+    ff_refstruct_pool_uninit(&pools->mbskip_table_pool);
+    ff_refstruct_pool_uninit(&pools->qscale_table_pool);
+    ff_refstruct_pool_uninit(&pools->mb_type_pool);
+    ff_refstruct_pool_uninit(&pools->motion_val_pool);
+    ff_refstruct_pool_uninit(&pools->ref_index_pool);
     pools->alloc_mb_height = pools->alloc_mb_width = pools->alloc_mb_stride = 0;
 }
 
@@ -641,15 +642,15 @@ int ff_mpv_init_context_frame(MpegEncContext *s)
         return AVERROR(ENOMEM);
     memset(s->mbintra_table, 1, mb_array_size);
 
-#define ALLOC_POOL(name, size) do { \
-    pools->name ##_pool = av_buffer_pool_init((size), av_buffer_allocz); \
+#define ALLOC_POOL(name, size, flags) do { \
+    pools->name ##_pool = ff_refstruct_pool_alloc((size), (flags)); \
     if (!pools->name ##_pool) \
         return AVERROR(ENOMEM); \
 } while (0)
 
-    ALLOC_POOL(mbskip_table, mb_array_size + 2);
-    ALLOC_POOL(qscale_table, mv_table_size);
-    ALLOC_POOL(mb_type, mv_table_size * sizeof(uint32_t));
+    ALLOC_POOL(mbskip_table, mb_array_size + 2, FF_REFSTRUCT_POOL_FLAG_ZERO_EVERY_TIME);
+    ALLOC_POOL(qscale_table, mv_table_size, 0);
+    ALLOC_POOL(mb_type, mv_table_size * sizeof(uint32_t), 0);
 
     if (s->out_format == FMT_H263 || s->encoding ||
         (s->avctx->export_side_data & AV_CODEC_EXPORT_DATA_MVS)) {
@@ -657,8 +658,11 @@ int ff_mpv_init_context_frame(MpegEncContext *s)
         int mv_size        = 2 * (b8_array_size + 4) * sizeof(int16_t);
         int ref_index_size = 4 * mb_array_size;
 
-        ALLOC_POOL(motion_val, mv_size);
-        ALLOC_POOL(ref_index, ref_index_size);
+        /* FIXME: The output of H.263 with OBMC depends upon
+         * the earlier content of the buffer; therefore we set
+         * the flags to always reset returned buffers here. */
+        ALLOC_POOL(motion_val, mv_size, FF_REFSTRUCT_POOL_FLAG_ZERO_EVERY_TIME);
+        ALLOC_POOL(ref_index, ref_index_size, 0);
     }
 #undef ALLOC_POOL
     pools->alloc_mb_width  = s->mb_width;
