diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index c8f9152451..7999ff6692 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -145,78 +145,6 @@ static void global_motion_params(AV1DecContext *s)
     }
 }
 
-static int get_relative_dist(const AV1RawSequenceHeader *seq,
-                             unsigned int a, unsigned int b)
-{
-    unsigned int diff = a - b;
-    unsigned int m = 1 << seq->order_hint_bits_minus_1;
-    return (diff & (m - 1)) - (diff & m);
-}
-
-static void skip_mode_params(AV1DecContext *s)
-{
-    const AV1RawFrameHeader *header = s->raw_frame_header;
-    const AV1RawSequenceHeader *seq = s->raw_seq;
-
-    int forward_idx,  backward_idx;
-    int forward_hint, backward_hint;
-    int second_forward_idx, second_forward_hint;
-    int ref_hint, dist, i;
-
-    if (!header->skip_mode_present)
-        return;
-
-    forward_idx  = -1;
-    backward_idx = -1;
-    for (i = 0; i < AV1_REFS_PER_FRAME; i++) {
-        ref_hint = s->ref[header->ref_frame_idx[i]].raw_frame_header->order_hint;
-        dist = get_relative_dist(seq, ref_hint, header->order_hint);
-        if (dist < 0) {
-            if (forward_idx < 0 ||
-                get_relative_dist(seq, ref_hint, forward_hint) > 0) {
-                forward_idx  = i;
-                forward_hint = ref_hint;
-            }
-        } else if (dist > 0) {
-            if (backward_idx < 0 ||
-                get_relative_dist(seq, ref_hint, backward_hint) < 0) {
-                backward_idx  = i;
-                backward_hint = ref_hint;
-            }
-        }
-    }
-
-    if (forward_idx < 0) {
-        return;
-    } else if (backward_idx >= 0) {
-        s->cur_frame.skip_mode_frame_idx[0] =
-            AV1_REF_FRAME_LAST + FFMIN(forward_idx, backward_idx);
-        s->cur_frame.skip_mode_frame_idx[1] =
-            AV1_REF_FRAME_LAST + FFMAX(forward_idx, backward_idx);
-        return;
-    }
-
-    second_forward_idx = -1;
-    for (i = 0; i < AV1_REFS_PER_FRAME; i++) {
-        ref_hint = s->ref[header->ref_frame_idx[i]].raw_frame_header->order_hint;
-        if (get_relative_dist(seq, ref_hint, forward_hint) < 0) {
-            if (second_forward_idx < 0 ||
-                get_relative_dist(seq, ref_hint, second_forward_hint) > 0) {
-                second_forward_idx  = i;
-                second_forward_hint = ref_hint;
-            }
-        }
-    }
-
-    if (second_forward_idx < 0)
-        return;
-
-    s->cur_frame.skip_mode_frame_idx[0] =
-        AV1_REF_FRAME_LAST + FFMIN(forward_idx, second_forward_idx);
-    s->cur_frame.skip_mode_frame_idx[1] =
-        AV1_REF_FRAME_LAST + FFMAX(forward_idx, second_forward_idx);
-}
-
 static int init_tile_data(AV1DecContext *s)
 
 {
@@ -415,8 +343,6 @@ static void av1_frame_unref(AVCodecContext *avctx, AV1Frame *f)
     av_buffer_unref(&f->header_ref);
     f->raw_frame_header = NULL;
     f->spatial_id = f->temporal_id = 0;
-    memset(f->skip_mode_frame_idx, 0,
-           2 * sizeof(uint8_t));
 }
 
 static int av1_frame_ref(AVCodecContext *avctx, AV1Frame *dst, const AV1Frame *src)
@@ -448,9 +374,6 @@ static int av1_frame_ref(AVCodecContext *avctx, AV1Frame *dst, const AV1Frame *s
     memcpy(dst->gm_params,
            src->gm_params,
            AV1_NUM_REF_FRAMES * 6 * sizeof(int32_t));
-    memcpy(dst->skip_mode_frame_idx,
-           src->skip_mode_frame_idx,
-           2 * sizeof(uint8_t));
 
     return 0;
 
@@ -727,7 +650,6 @@ static int get_current_frame(AVCodecContext *avctx)
     }
 
     global_motion_params(s);
-    skip_mode_params(s);
 
     return ret;
 }
diff --git a/libavcodec/av1dec.h b/libavcodec/av1dec.h
index bc6ae68d1a..7f07aa9f7a 100644
--- a/libavcodec/av1dec.h
+++ b/libavcodec/av1dec.h
@@ -44,8 +44,6 @@ typedef struct AV1Frame {
 
     uint8_t gm_type[AV1_NUM_REF_FRAMES];
     int32_t gm_params[AV1_NUM_REF_FRAMES][6];
-
-    uint8_t skip_mode_frame_idx[2];
 } AV1Frame;
 
 typedef struct TileGroupInfo {
diff --git a/libavcodec/nvdec_av1.c b/libavcodec/nvdec_av1.c
index 1a553902bc..d1161dd258 100644
--- a/libavcodec/nvdec_av1.c
+++ b/libavcodec/nvdec_av1.c
@@ -64,6 +64,68 @@ static int get_coded_lossless_param(const AV1DecContext *s)
     return 1;
 }
 
+static int get_relative_dist(const AV1RawSequenceHeader *seq,
+                             unsigned int a, unsigned int b)
+{
+    unsigned int diff = a - b;
+    unsigned int m = 1 << seq->order_hint_bits_minus_1;
+    return (diff & (m - 1)) - (diff & m);
+}
+
+static void skip_mode_params(const AV1DecContext *s, uint8_t *skip_mode_frame_idx)
+{
+    const AV1RawFrameHeader *frame_header = s->raw_frame_header;
+    const AV1RawSequenceHeader *seq = s->raw_seq;
+
+    int forward_idx = -1, backward_idx = -1;
+    int forward_hint, backward_hint;
+    int second_forward_idx, second_forward_hint;
+
+    for (int i = 0; i < AV1_REFS_PER_FRAME; i++) {
+        int ref_hint = s->ref[frame_header->ref_frame_idx[i]].raw_frame_header->order_hint;
+        int dist = get_relative_dist(seq, ref_hint, frame_header->order_hint);
+        if (dist < 0) {
+            if (forward_idx < 0 ||
+                get_relative_dist(seq, ref_hint, forward_hint) > 0) {
+                forward_idx  = i;
+                forward_hint = ref_hint;
+            }
+        } else if (dist > 0) {
+            if (backward_idx < 0 ||
+                get_relative_dist(seq, ref_hint, backward_hint) < 0) {
+                backward_idx  = i;
+                backward_hint = ref_hint;
+            }
+        }
+    }
+
+    if (forward_idx < 0) {
+        return;
+    } else if (backward_idx >= 0) {
+        skip_mode_frame_idx[0] = AV1_REF_FRAME_LAST + FFMIN(forward_idx, backward_idx);
+        skip_mode_frame_idx[1] = AV1_REF_FRAME_LAST + FFMAX(forward_idx, backward_idx);
+        return;
+    }
+
+    second_forward_idx = -1;
+    for (int i = 0; i < AV1_REFS_PER_FRAME; i++) {
+        int ref_hint = s->ref[frame_header->ref_frame_idx[i]].raw_frame_header->order_hint;
+        if (get_relative_dist(seq, ref_hint, forward_hint) < 0) {
+            if (second_forward_idx < 0 ||
+                get_relative_dist(seq, ref_hint, second_forward_hint) > 0) {
+                second_forward_idx  = i;
+                second_forward_hint = ref_hint;
+            }
+        }
+    }
+
+    if (second_forward_idx < 0)
+        return;
+
+    skip_mode_frame_idx[0] = AV1_REF_FRAME_LAST + FFMIN(forward_idx, second_forward_idx);
+    skip_mode_frame_idx[1] = AV1_REF_FRAME_LAST + FFMAX(forward_idx, second_forward_idx);
+}
+
 static int nvdec_av1_start_frame(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size)
 {
     const AV1DecContext *s = avctx->priv_data;
@@ -78,6 +140,7 @@ static int nvdec_av1_start_frame(AVCodecContext *avctx, const uint8_t *buffer, u
     AVFrame *cur_frame = s->cur_frame.tf.f;
 
     unsigned char remap_lr_type[4] = { AV1_RESTORE_NONE, AV1_RESTORE_SWITCHABLE, AV1_RESTORE_WIENER, AV1_RESTORE_SGRPROJ };
+    uint8_t skip_mode_frame_idx[2] = { 0 };
 
     int ret, i, j;
 
@@ -90,6 +153,9 @@ static int nvdec_av1_start_frame(AVCodecContext *avctx, const uint8_t *buffer, u
     else
         fg_header = frame_header;
 
+    if (frame_header->skip_mode_present)
+        skip_mode_params(s, skip_mode_frame_idx);
+
     fdd = (FrameDecodeData*)cur_frame->private_ref->data;
     cf  = (NVDECFrame*)fdd->hwaccel_priv;
 
@@ -164,10 +230,8 @@ static int nvdec_av1_start_frame(AVCodecContext *avctx, const uint8_t *buffer, u
             .cdef_bits            = frame_header->cdef_bits,
 
             /* SkipModeFrames */
-            .SkipModeFrame0 = frame_header->skip_mode_present ?
-                              s->cur_frame.skip_mode_frame_idx[0] : 0,
-            .SkipModeFrame1 = frame_header->skip_mode_present ?
-                              s->cur_frame.skip_mode_frame_idx[1] : 0,
+            .SkipModeFrame0 = skip_mode_frame_idx[0],
+            .SkipModeFrame1 = skip_mode_frame_idx[1],
 
             /* QP Information */
             .base_qindex     = frame_header->base_q_idx,
