diff --git a/ffmpeg_qsv.c b/ffmpeg_qsv.c
index 86824b6..8cedb7e 100644
--- a/ffmpeg_qsv.c
+++ b/ffmpeg_qsv.c
@@ -81,25 +81,26 @@ int qsv_init(AVCodecContext *s)
             return ret;
     }
 
-    av_buffer_unref(&ist->hw_frames_ctx);
-    ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
-    if (!ist->hw_frames_ctx)
-        return AVERROR(ENOMEM);
-
-    frames_ctx   = (AVHWFramesContext*)ist->hw_frames_ctx->data;
-    frames_hwctx = frames_ctx->hwctx;
-
-    frames_ctx->width             = FFALIGN(s->coded_width,  32);
-    frames_ctx->height            = FFALIGN(s->coded_height, 32);
-    frames_ctx->format            = AV_PIX_FMT_QSV;
-    frames_ctx->sw_format         = s->sw_pix_fmt;
-    frames_ctx->initial_pool_size = 64;
-    frames_hwctx->frame_type      = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
-
-    ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
-    if (ret < 0) {
-        av_log(NULL, AV_LOG_ERROR, "Error initializing a QSV frame pool\n");
-        return ret;
+    if(!ist->hw_frames_ctx) {
+        ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
+        if (!ist->hw_frames_ctx)
+            return AVERROR(ENOMEM);
+
+        frames_ctx   = (AVHWFramesContext*)ist->hw_frames_ctx->data;
+        frames_hwctx = frames_ctx->hwctx;
+
+        frames_ctx->width             = FFALIGN(s->coded_width,  32);
+        frames_ctx->height            = FFALIGN(s->coded_height, 32);
+        frames_ctx->format            = AV_PIX_FMT_QSV;
+        frames_ctx->sw_format         = s->sw_pix_fmt;
+        frames_ctx->initial_pool_size = 64;
+        frames_hwctx->frame_type      = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
+
+        ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
+        if (ret < 0) {
+            av_log(NULL, AV_LOG_ERROR, "Error initializing a QSV frame pool\n");
+            return ret;
+        }
     }
 
     ist->hwaccel_get_buffer = qsv_get_buffer;
@@ -114,9 +115,8 @@ int qsv_transcode_init(OutputStream *ost)
     const enum AVPixelFormat *pix_fmt;
 
     int err, i;
-    AVBufferRef *encode_frames_ref = NULL;
-    AVHWFramesContext *encode_frames;
-    AVQSVFramesContext *qsv_frames;
+    AVHWFramesContext *frames_ctx;
+    AVQSVFramesContext *frames_hwctx;
 
     /* check if the encoder supports QSV */
     if (!ost->enc->pix_fmts)
@@ -150,42 +150,33 @@ int qsv_transcode_init(OutputStream *ost)
     if (!hw_device_ctx) {
         err = qsv_device_init(ist);
         if (err < 0)
-            goto fail;
-    }
-
-    // This creates a dummy hw_frames_ctx for the encoder to be
-    // suitably initialised.  It only contains one real frame, so
-    // hopefully doesn't waste too much memory.
-
-    encode_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx);
-    if (!encode_frames_ref) {
-        err = AVERROR(ENOMEM);
-        goto fail;
+            return err;
     }
-    encode_frames = (AVHWFramesContext*)encode_frames_ref->data;
-    qsv_frames = encode_frames->hwctx;
 
-    encode_frames->width     = FFALIGN(ist->resample_width,  32);
-    encode_frames->height    = FFALIGN(ist->resample_height, 32);
-    encode_frames->format    = AV_PIX_FMT_QSV;
-    encode_frames->sw_format = AV_PIX_FMT_NV12;
-    encode_frames->initial_pool_size = 1;
+    ist->dec_ctx->pix_fmt = AV_PIX_FMT_QSV;
+    ist->resample_pix_fmt = AV_PIX_FMT_QSV;
+    ist->hw_frames_ctx    = av_hwframe_ctx_alloc(hw_device_ctx);
+    if (!ist->hw_frames_ctx)
+        return AVERROR(ENOMEM);
 
-    qsv_frames->frame_type = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
+    frames_ctx   = (AVHWFramesContext*)ist->hw_frames_ctx->data;
+    frames_hwctx = frames_ctx->hwctx;
 
-    err = av_hwframe_ctx_init(encode_frames_ref);
-    if (err < 0)
-        goto fail;
+    frames_ctx->width             = FFALIGN(ist->resample_width,  32);
+    frames_ctx->height            = FFALIGN(ist->resample_height, 32);
+    frames_ctx->format            = AV_PIX_FMT_QSV;
+    frames_ctx->sw_format         = AV_PIX_FMT_NV12;
+    frames_ctx->initial_pool_size = 64;
+    frames_hwctx->frame_type      = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
 
-    ist->dec_ctx->pix_fmt       = AV_PIX_FMT_QSV;
-    ist->resample_pix_fmt       = AV_PIX_FMT_QSV;
+    err = av_hwframe_ctx_init(ist->hw_frames_ctx);
+    if (err < 0) {
+        av_buffer_unref(&ist->hw_frames_ctx);
+        av_log(NULL, AV_LOG_ERROR, "Error initializing a QSV frame pool\n");
+        return err;
+    }
 
-    ost->enc_ctx->pix_fmt       = AV_PIX_FMT_QSV;
-    ost->enc_ctx->hw_frames_ctx = encode_frames_ref;
+    ost->enc_ctx->pix_fmt = AV_PIX_FMT_QSV;
 
     return 0;
-
-fail:
-    av_buffer_unref(&encode_frames_ref);
-    return err;
 }
