diff --git a/libavcodec/ac3enc.c b/libavcodec/ac3enc.c
index ef1ac381c1..681b227d3d 100644
--- a/libavcodec/ac3enc.c
+++ b/libavcodec/ac3enc.c
@@ -52,6 +52,9 @@
 #include "ac3enc.h"
 #include "eac3enc.h"
 
+#define SAMPLETYPE_SIZE(ctx) (sizeof(float) == sizeof(int32_t) ? sizeof(float) : \
+                                  (ctx)->fixed_point ? sizeof(int32_t) : sizeof(float))
+
 typedef struct AC3Mant {
     int16_t *qmant1_ptr, *qmant2_ptr, *qmant4_ptr; ///< mantissa pointers for bap=1,2,4
     int mant1_cnt, mant2_cnt, mant4_cnt;    ///< mantissa counts for bap=1,2,4
@@ -2429,10 +2432,21 @@ static av_cold int allocate_buffers(AC3EncodeContext *s)
     int channels = s->channels + 1; /* includes coupling channel */
     int channel_blocks = channels * s->num_blocks;
     int total_coefs    = AC3_MAX_COEFS * channel_blocks;
+    const unsigned sampletype_size = SAMPLETYPE_SIZE(s);
+
+    if (!(s->windowed_samples = av_malloc(sampletype_size * AC3_WINDOW_SIZE)))
+        return AVERROR(ENOMEM);
 
-    if (s->allocate_sample_buffers(s))
+    if (!FF_ALLOCZ_TYPED_ARRAY(s->planar_samples,  s->channels))
         return AVERROR(ENOMEM);
 
+    for (int ch = 0; ch < s->channels; ch++) {
+        s->planar_samples[ch] = av_mallocz((AC3_FRAME_SIZE + AC3_BLOCK_SIZE) *
+                                                  sampletype_size);
+        if (!s->planar_samples[ch])
+            return AVERROR(ENOMEM);
+    }
+
     if (!FF_ALLOC_TYPED_ARRAY(s->bap_buffer,         total_coefs)          ||
         !FF_ALLOC_TYPED_ARRAY(s->bap1_buffer,        total_coefs)          ||
         !FF_ALLOCZ_TYPED_ARRAY(s->mdct_coef_buffer,  total_coefs)          ||
diff --git a/libavcodec/ac3enc.h b/libavcodec/ac3enc.h
index 3dc8acfd91..8d6cb3b3de 100644
--- a/libavcodec/ac3enc.h
+++ b/libavcodec/ac3enc.h
@@ -232,8 +232,8 @@ typedef struct AC3EncodeContext {
     int frame_bits;                         ///< all frame bits except exponents and mantissas
     int exponent_bits;                      ///< number of bits used for exponents
 
-    SampleType *windowed_samples;
-    SampleType **planar_samples;
+    void *windowed_samples;
+    uint8_t **planar_samples;
     uint8_t *bap_buffer;
     uint8_t *bap1_buffer;
     CoefType *mdct_coef_buffer;
@@ -260,9 +260,6 @@ typedef struct AC3EncodeContext {
     /* fixed vs. float function pointers */
     int  (*mdct_init)(struct AC3EncodeContext *s);
 
-    /* fixed vs. float templated function pointers */
-    int  (*allocate_sample_buffers)(struct AC3EncodeContext *s);
-
     /* AC-3 vs. E-AC-3 function pointers */
     void (*output_frame_header)(struct AC3EncodeContext *s);
 } AC3EncodeContext;
diff --git a/libavcodec/ac3enc_fixed.c b/libavcodec/ac3enc_fixed.c
index 4a24cce833..b8a4d88a42 100644
--- a/libavcodec/ac3enc_fixed.c
+++ b/libavcodec/ac3enc_fixed.c
@@ -104,7 +104,6 @@ static av_cold int ac3_fixed_encode_init(AVCodecContext *avctx)
     s->fixed_point = 1;
     s->encode_frame            = encode_frame;
     s->mdct_init               = ac3_fixed_mdct_init;
-    s->allocate_sample_buffers = allocate_sample_buffers;
     return ff_ac3_encode_init(avctx);
 }
 
diff --git a/libavcodec/ac3enc_float.c b/libavcodec/ac3enc_float.c
index e0907fed05..77a7725f31 100644
--- a/libavcodec/ac3enc_float.c
+++ b/libavcodec/ac3enc_float.c
@@ -107,7 +107,6 @@ av_cold int ff_ac3_float_encode_init(AVCodecContext *avctx)
 
     s->encode_frame            = encode_frame;
     s->mdct_init               = ac3_float_mdct_init;
-    s->allocate_sample_buffers = allocate_sample_buffers;
     s->fdsp = avpriv_float_dsp_alloc(avctx->flags & AV_CODEC_FLAG_BITEXACT);
     if (!s->fdsp)
         return AVERROR(ENOMEM);
diff --git a/libavcodec/ac3enc_template.c b/libavcodec/ac3enc_template.c
index 56ce36c012..3646e1dcaa 100644
--- a/libavcodec/ac3enc_template.c
+++ b/libavcodec/ac3enc_template.c
@@ -31,8 +31,6 @@
 #include <stdint.h>
 
 #include "libavutil/attributes.h"
-#include "libavutil/internal.h"
-#include "libavutil/mem.h"
 #include "libavutil/mem_internal.h"
 
 #include "audiodsp.h"
@@ -40,23 +38,6 @@
 #include "eac3enc.h"
 
 
-static int allocate_sample_buffers(AC3EncodeContext *s)
-{
-    int ch;
-
-    if (!FF_ALLOC_TYPED_ARRAY(s->windowed_samples, AC3_WINDOW_SIZE) ||
-        !FF_ALLOCZ_TYPED_ARRAY(s->planar_samples,  s->channels))
-        return AVERROR(ENOMEM);
-
-    for (ch = 0; ch < s->channels; ch++) {
-        if (!(s->planar_samples[ch] = av_mallocz((AC3_FRAME_SIZE + AC3_BLOCK_SIZE) *
-                                                  sizeof(**s->planar_samples))))
-            return AVERROR(ENOMEM);
-    }
-    return 0;
-}
-
-
 /*
  * Copy input samples.
  * Channels are reordered from FFmpeg's default order to AC-3 order.
@@ -68,13 +49,14 @@ static void copy_input_samples(AC3EncodeContext *s, SampleType **samples)
     /* copy and remap input samples */
     for (ch = 0; ch < s->channels; ch++) {
         /* copy last 256 samples of previous frame to the start of the current frame */
-        memcpy(&s->planar_samples[ch][0], &s->planar_samples[ch][AC3_BLOCK_SIZE * s->num_blocks],
-               AC3_BLOCK_SIZE * sizeof(s->planar_samples[0][0]));
+        memcpy(&s->planar_samples[ch][0],
+               (SampleType*)s->planar_samples[ch] + AC3_BLOCK_SIZE * s->num_blocks,
+               AC3_BLOCK_SIZE * sizeof(SampleType));
 
         /* copy new samples for current frame */
-        memcpy(&s->planar_samples[ch][AC3_BLOCK_SIZE],
+        memcpy((SampleType*)s->planar_samples[ch] + AC3_BLOCK_SIZE,
                samples[s->channel_map[ch]],
-               AC3_BLOCK_SIZE * s->num_blocks * sizeof(s->planar_samples[0][0]));
+               AC3_BLOCK_SIZE * s->num_blocks * sizeof(SampleType));
     }
 }
 
@@ -91,11 +73,11 @@ static void apply_mdct(AC3EncodeContext *s)
     for (ch = 0; ch < s->channels; ch++) {
         for (blk = 0; blk < s->num_blocks; blk++) {
             AC3Block *block = &s->blocks[blk];
-            const SampleType *input_samples = &s->planar_samples[ch][blk * AC3_BLOCK_SIZE];
+            const SampleType *input_samples = (SampleType*)s->planar_samples[ch] + blk * AC3_BLOCK_SIZE;
 
             s->fdsp->vector_fmul(s->windowed_samples, input_samples,
                                  s->mdct_window, AC3_BLOCK_SIZE);
-            s->fdsp->vector_fmul_reverse(s->windowed_samples + AC3_BLOCK_SIZE,
+            s->fdsp->vector_fmul_reverse((SampleType*)s->windowed_samples + AC3_BLOCK_SIZE,
                                          &input_samples[AC3_BLOCK_SIZE],
                                          s->mdct_window, AC3_BLOCK_SIZE);
 
