diff --git a/libavcodec/hevc_refs.c b/libavcodec/hevc_refs.c
index ae464e8e6d..fa53b273c7 100644
--- a/libavcodec/hevc_refs.c
+++ b/libavcodec/hevc_refs.c
@@ -42,13 +42,11 @@ void ff_hevc_unref_frame(HEVCContext *s, HEVCFrame *frame, int flags)
         ff_thread_release_buffer(s->avctx, frame->frame_grain);
         frame->needs_fg = 0;
 
-        av_buffer_unref(&frame->tab_mvf_buf);
-        frame->tab_mvf = NULL;
+        ff_refstruct_unref(&frame->tab_mvf);
 
         ff_refstruct_unref(&frame->rpl);
         frame->nb_rpl_elems = 0;
-        av_buffer_unref(&frame->rpl_tab_buf);
-        frame->rpl_tab    = NULL;
+        ff_refstruct_unref(&frame->rpl_tab);
         frame->refPicList = NULL;
 
         frame->collocated_ref = NULL;
@@ -101,15 +99,13 @@ static HEVCFrame *alloc_frame(HEVCContext *s)
             goto fail;
         frame->nb_rpl_elems = s->pkt.nb_nals;
 
-        frame->tab_mvf_buf = av_buffer_pool_get(s->tab_mvf_pool);
-        if (!frame->tab_mvf_buf)
+        frame->tab_mvf = ff_refstruct_pool_get(s->tab_mvf_pool);
+        if (!frame->tab_mvf)
             goto fail;
-        frame->tab_mvf = (MvField *)frame->tab_mvf_buf->data;
 
-        frame->rpl_tab_buf = av_buffer_pool_get(s->rpl_tab_pool);
-        if (!frame->rpl_tab_buf)
+        frame->rpl_tab = ff_refstruct_pool_get(s->rpl_tab_pool);
+        if (!frame->rpl_tab)
             goto fail;
-        frame->rpl_tab   = (RefPicListTab **)frame->rpl_tab_buf->data;
         frame->ctb_count = s->ps.sps->ctb_width * s->ps.sps->ctb_height;
         for (j = 0; j < frame->ctb_count; j++)
             frame->rpl_tab[j] = frame->rpl;
diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c
index 44561de821..23cc543f82 100644
--- a/libavcodec/hevcdec.c
+++ b/libavcodec/hevcdec.c
@@ -86,8 +86,8 @@ static void pic_arrays_free(HEVCContext *s)
     av_freep(&s->sh.size);
     av_freep(&s->sh.offset);
 
-    av_buffer_pool_uninit(&s->tab_mvf_pool);
-    av_buffer_pool_uninit(&s->rpl_tab_pool);
+    ff_refstruct_pool_uninit(&s->tab_mvf_pool);
+    ff_refstruct_pool_uninit(&s->rpl_tab_pool);
 }
 
 /* allocate arrays that depend on frame dimensions */
@@ -133,10 +133,8 @@ static int pic_arrays_init(HEVCContext *s, const HEVCSPS *sps)
     if (!s->horizontal_bs || !s->vertical_bs)
         goto fail;
 
-    s->tab_mvf_pool = av_buffer_pool_init(min_pu_size * sizeof(MvField),
-                                          av_buffer_allocz);
-    s->rpl_tab_pool = av_buffer_pool_init(ctb_count * sizeof(RefPicListTab),
-                                          av_buffer_allocz);
+    s->tab_mvf_pool = ff_refstruct_pool_alloc(min_pu_size * sizeof(MvField), 0);
+    s->rpl_tab_pool = ff_refstruct_pool_alloc(ctb_count * sizeof(RefPicListTab), 0);
     if (!s->tab_mvf_pool || !s->rpl_tab_pool)
         goto fail;
 
@@ -3394,16 +3392,8 @@ static int hevc_ref_frame(HEVCContext *s, HEVCFrame *dst, HEVCFrame *src)
         dst->needs_fg = 1;
     }
 
-    dst->tab_mvf_buf = av_buffer_ref(src->tab_mvf_buf);
-    if (!dst->tab_mvf_buf)
-        goto fail;
-    dst->tab_mvf = src->tab_mvf;
-
-    dst->rpl_tab_buf = av_buffer_ref(src->rpl_tab_buf);
-    if (!dst->rpl_tab_buf)
-        goto fail;
-    dst->rpl_tab = src->rpl_tab;
-
+    dst->tab_mvf = ff_refstruct_ref(src->tab_mvf);
+    dst->rpl_tab = ff_refstruct_ref(src->rpl_tab);
     dst->rpl = ff_refstruct_ref(src->rpl);
     dst->nb_rpl_elems = src->nb_rpl_elems;
 
@@ -3416,9 +3406,6 @@ static int hevc_ref_frame(HEVCContext *s, HEVCFrame *dst, HEVCFrame *src)
                           src->hwaccel_picture_private);
 
     return 0;
-fail:
-    ff_hevc_unref_frame(s, dst, ~0);
-    return AVERROR(ENOMEM);
 }
 
 static av_cold int hevc_decode_free(AVCodecContext *avctx)
diff --git a/libavcodec/hevcdec.h b/libavcodec/hevcdec.h
index c13406f0a8..edf2f188cf 100644
--- a/libavcodec/hevcdec.h
+++ b/libavcodec/hevcdec.h
@@ -408,15 +408,13 @@ typedef struct HEVCFrame {
     AVFrame *frame_grain;
     ThreadFrame tf;
     int needs_fg; /* 1 if grain needs to be applied by the decoder */
-    MvField *tab_mvf;
+    MvField *tab_mvf;              ///< RefStruct reference
     RefPicList *refPicList;
-    RefPicListTab **rpl_tab;
+    RefPicListTab **rpl_tab;       ///< RefStruct reference
     int ctb_count;
     int poc;
     struct HEVCFrame *collocated_ref;
 
-    AVBufferRef *tab_mvf_buf;
-    AVBufferRef *rpl_tab_buf;
     RefPicListTab *rpl;            ///< RefStruct reference
     int nb_rpl_elems;
 
@@ -517,8 +515,8 @@ typedef struct HEVCContext {
     HEVCSEI sei;
     struct AVMD5 *md5_ctx;
 
-    AVBufferPool *tab_mvf_pool;
-    AVBufferPool *rpl_tab_pool;
+    struct FFRefStructPool *tab_mvf_pool;
+    struct FFRefStructPool *rpl_tab_pool;
 
     ///< candidate references for the current frame
     RefPicList rps[5];
