diff --git a/libavcodec/vorbis_enc_data.h b/libavcodec/vorbis_enc_data.h
index eca43dfded..6f2b10feb9 100644
--- a/libavcodec/vorbis_enc_data.h
+++ b/libavcodec/vorbis_enc_data.h
@@ -23,6 +23,26 @@
 
 #include <stdint.h>
 
+typedef const struct {
+    int dim;
+    int len;
+    int real_len;
+    const uint8_t *clens;
+    int lookup;
+    float min;
+    float delta;
+    const uint8_t *quant;
+} codebook_setup;
+
+typedef const struct {
+    const int type;
+    const int end[2];
+    const int classifications;
+    const int nbooks;
+    const codebook_setup *config;
+    const int8_t books[10][8];
+} res_setup;
+
 static const uint8_t floor_128_c0[] = {
     10,  7,  8, 13,  9,  6,  7, 11, 10,  8,  8, 12, 17, 17, 17,
     17,  7,  5,  5,  9,  6,  4,  4,  8,  8,  5,  5,  8, 16, 14,
@@ -219,7 +239,7 @@ static const uint8_t floor_1024_4sub3[] = {
     11, 11, 10, 10, 10, 10, 10, 10,
 };
 
-static const uint8_t res_long_master[] = {
+static const uint8_t res_stereo_long_master[] = {
      5,  6, 11, 11, 11, 11, 10, 10, 12, 11,  5,  2, 11,  5,  6,
      6,  7,  9, 11, 13, 13, 10,  7, 11,  6,  7,  8,  9, 10, 12,
     11,  5, 11,  6,  8,  7,  9, 11, 14, 15, 11,  6,  6,  8,  4,
@@ -229,7 +249,7 @@ static const uint8_t res_long_master[] = {
     11, 13, 12, 15, 12, 11,  9,  8,  8,  8,
 };
 
-static const uint8_t res_short_master[] = {
+static const uint8_t res_stereo_short_master[] = {
     10,  9, 13, 11, 14, 10, 12, 13, 13, 14,  7,  2, 12,  5, 10,
      5,  7, 10, 12, 14, 12,  6,  9,  8,  7,  7,  9, 11, 13, 16,
     10,  4, 12,  5, 10,  6,  8, 12, 14, 16, 12,  6,  8,  7,  6,
@@ -239,7 +259,7 @@ static const uint8_t res_short_master[] = {
     15, 13, 11, 10,  6,  5,  6,  8,  9, 11,
 };
 
-static const uint8_t res_p1_0[] = {
+static const uint8_t res_stereo_p1_0[] = {
      2,  4,  4,  0,  0,  0,  0,  0,  0,  5,  6,  6,  0,  0,  0,
      0,  0,  0,  5,  6,  6,  0,  0,  0,  0,  0,  0,  0,  0,  0,
      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
@@ -352,7 +372,7 @@ static const uint8_t res_p1_0[] = {
      0,  0,  0,  8,  9,  8,
 };
 
-static const uint8_t res_p2_0[] = {
+static const uint8_t res_stereo_p2_0[] = {
      2,  5,  5,  0,  0,  0,  5,  5,  0,  0,  0,  5,  5,  0,  0,
      0,  7,  8,  0,  0,  0,  0,  0,  0,  0,  5,  6,  6,  0,  0,
      0,  7,  7,  0,  0,  0,  7,  7,  0,  0,  0, 10, 10,  0,  0,
@@ -385,7 +405,7 @@ static const uint8_t res_p2_0[] = {
      0,  9,  9,  0,  0,  0, 10, 10,
 };
 
-static const uint8_t res_p3_0[] = {
+static const uint8_t res_stereo_p3_0[] = {
      2,  4,  3,  6,  6,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  4,  4,  4,  6,  6,
      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
@@ -395,7 +415,7 @@ static const uint8_t res_p3_0[] = {
      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  6,  6,  7,  9,  9,
 };
 
-static const uint8_t res_p4_0[] = {
+static const uint8_t res_stereo_p4_0[] = {
      2,  3,  3,  6,  6,  0,  0,  0,  0,  0,  4,  4,  6,  6,  0,
      0,  0,  0,  0,  4,  4,  6,  6,  0,  0,  0,  0,  0,  5,  5,
      6,  6,  0,  0,  0,  0,  0,  0,  0,  6,  6,  0,  0,  0,  0,
@@ -403,7 +423,7 @@ static const uint8_t res_p4_0[] = {
      0,  0,  0,  0,  0,  0,  9,  9,
 };
 
-static const uint8_t res_p5_0[] = {
+static const uint8_t res_stereo_p5_0[] = {
      1,  3,  4,  6,  6,  7,  7,  9,  9,  0,  5,  5,  7,  7,  7,
      8,  9,  9,  0,  5,  5,  7,  7,  8,  8,  9,  9,  0,  7,  7,
      8,  8,  8,  8, 10, 10,  0,  0,  0,  8,  8,  8,  8, 10, 10,
@@ -412,7 +432,7 @@ static const uint8_t res_p5_0[] = {
      0,  0, 10, 10, 11, 11,
 };
 
-static const uint8_t res_p6_0[] = {
+static const uint8_t res_stereo_p6_0[] = {
      2,  3,  3,  6,  6,  7,  7,  8,  8,  8,  8,  9,  9, 10, 10,
     11, 10,  0,  5,  5,  7,  7,  8,  8,  9,  9,  9,  9, 10, 10,
     10, 10, 11, 11,  0,  5,  5,  7,  7,  8,  8,  9,  9,  9,  9,
@@ -435,7 +455,7 @@ static const uint8_t res_p6_0[] = {
     13, 13, 13, 13,
 };
 
-static const uint8_t res_p7_0[] = {
+static const uint8_t res_stereo_p7_0[] = {
      1,  4,  4,  7,  6,  6,  7,  6,  6,  4,  7,  7, 10,  9,  9,
     11,  9,  9,  4,  7,  7, 10,  9,  9, 11,  9,  9,  7, 10, 10,
     11, 11, 10, 12, 11, 11,  6,  9,  9, 11, 10, 10, 11, 10, 10,
@@ -444,7 +464,7 @@ static const uint8_t res_p7_0[] = {
     11, 10, 10, 11, 10, 10,
 };
 
-static const uint8_t res_p7_1[] = {
+static const uint8_t res_stereo_p7_1[] = {
      2,  4,  4,  6,  6,  7,  7,  7,  7,  8,  8, 10,  5,  5,  6,
      6,  7,  7,  8,  8,  8,  8, 10,  5,  5,  6,  6,  7,  7,  8,
      8,  8,  8, 10,  6,  6,  7,  7,  8,  8,  8,  8,  8,  8, 10,
@@ -456,7 +476,7 @@ static const uint8_t res_p7_1[] = {
      8,
 };
 
-static const uint8_t res_p8_0[] = {
+static const uint8_t res_stereo_p8_0[] = {
      1,  4,  4,  6,  6,  7,  7,  8,  8,  9,  9, 10, 10,  6,  5,
      5,  7,  7,  8,  8,  8,  8,  9,  9, 10, 10,  7,  5,  5,  7,
      7,  8,  8,  8,  8,  9,  9, 11, 10,  0,  8,  8,  8,  8,  9,
@@ -471,12 +491,12 @@ static const uint8_t res_p8_0[] = {
     13, 12, 14, 13,
 };
 
-static const uint8_t res_p8_1[] = {
+static const uint8_t res_stereo_p8_1[] = {
      2,  4,  4,  5,  5,  6,  5,  5,  5,  5,  6,  4,  5,  5,  5,
      6,  5,  5,  5,  5,  6,  6,  6,  5,  5,
 };
 
-static const uint8_t res_p9_0[] = {
+static const uint8_t res_stereo_p9_0[] = {
      1,  4,  4, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,  4,  9,
      8, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,  2,  9,  7, 12,
     12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
@@ -491,7 +511,7 @@ static const uint8_t res_p9_0[] = {
     11, 11, 11, 11,
 };
 
-static const uint8_t res_p9_1[] = {
+static const uint8_t res_stereo_p9_1[] = {
      1,  4,  4,  6,  6,  7,  7,  8,  7,  9,  9, 10, 10, 10, 10,
      6,  5,  5,  7,  7,  8,  8, 10,  8, 11, 10, 12, 12, 13, 13,
      6,  5,  5,  7,  7,  8,  8, 10,  9, 11, 11, 12, 12, 13, 12,
@@ -509,7 +529,7 @@ static const uint8_t res_p9_1[] = {
     18, 18, 18, 16, 17, 16, 14, 12, 11, 13, 10, 13, 13, 14, 15,
 };
 
-static const uint8_t res_p9_2[] = {
+static const uint8_t res_stereo_p9_2[] = {
      2,  5,  5,  6,  6,  7,  7,  7,  7,  7,  7,  8,  8,  8,  8,
      8,  8, 10,  6,  6,  7,  7,  8,  7,  8,  8,  8,  8,  8,  9,
      9,  9,  9,  9, 10,  6,  6,  7,  7,  7,  7,  8,  8,  8,  8,
@@ -532,16 +552,7 @@ static const uint8_t res_p9_2[] = {
      9, 10, 10, 10,
 };
 
-static const struct {
-    int dim;
-    int len;
-    int real_len;
-    const uint8_t *clens;
-    int lookup;
-    float min;
-    float delta;
-    const uint8_t *quant;
-} cvectors[] = {
+codebook_setup floor_config[] = {
     { 2,   64,   64, floor_128_c0,     0 },
     { 2,  256,  256, floor_128_c1,     0 },
     { 2,   16,   16, floor_1024_c1,    0 },
@@ -565,21 +576,24 @@ static const struct {
     { 2,   18,   18, floor_1024_4sub1, 0 },
     { 2,   50,   50, floor_1024_4sub2, 0 },
     { 2,  128,  128, floor_1024_4sub3, 0 },
-    { 2,  100,  100, res_short_master, 0 },
-    { 2,  100,  100, res_long_master,  0 },
-    { 8, 1641, 6561, res_p1_0, 1,    -1.0,   1.0, (const uint8_t[]){ 1, 0, 2, } },
-    { 4,  443,  625, res_p2_0, 1,    -2.0,   1.0, (const uint8_t[]){ 2, 1, 3, 0, 4, } },
-    { 4,  105,  625, res_p3_0, 1,    -2.0,   1.0, (const uint8_t[]){ 2, 1, 3, 0, 4, } },
-    { 2,   68,   81, res_p4_0, 1,    -4.0,   1.0, (const uint8_t[]){ 4, 3, 5, 2, 6, 1, 7, 0, 8, } },
-    { 2,   81,   81, res_p5_0, 1,    -4.0,   1.0, (const uint8_t[]){ 4, 3, 5, 2, 6, 1, 7, 0, 8, } },
-    { 2,  289,  289, res_p6_0, 1,    -8.0,   1.0, (const uint8_t[]){ 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, 0, 16, } },
-    { 4,   81,   81, res_p7_0, 1,   -11.0,  11.0, (const uint8_t[]){ 1, 0, 2, } },
-    { 2,  121,  121, res_p7_1, 1,    -5.0,   1.0, (const uint8_t[]){ 5, 4, 6, 3, 7, 2, 8, 1, 9, 0, 10, } },
-    { 2,  169,  169, res_p8_0, 1,   -30.0,   5.0, (const uint8_t[]){ 6, 5, 7, 4, 8, 3, 9, 2, 10, 1, 11, 0, 12, } },
-    { 2,   25,   25, res_p8_1, 1,    -2.0,   1.0, (const uint8_t[]){ 2, 1, 3, 0, 4, } },
-    { 2,  169,  169, res_p9_0, 1, -1530.0, 255.0, (const uint8_t[]){ 6, 5, 7, 4, 8, 3, 9, 2, 10, 1, 11, 0, 12, } },
-    { 2,  225,  225, res_p9_1, 1,  -119.0,  17.0, (const uint8_t[]){ 7, 6, 8, 5, 9, 4, 10, 3, 11, 2, 12, 1, 13, 0, 14, } },
-    { 2,  289,  289, res_p9_2, 1,    -8.0,   1.0, (const uint8_t[]){ 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, 0, 16, } },
+};
+
+static codebook_setup res_stereo_config[] = {
+    { 2,  100,  100, res_stereo_short_master, 0 },
+    { 2,  100,  100, res_stereo_long_master,  0 },
+    { 8, 1641, 6561, res_stereo_p1_0, 1,    -1.0,   1.0, (const uint8_t[]){ 1, 0, 2, } },
+    { 4,  443,  625, res_stereo_p2_0, 1,    -2.0,   1.0, (const uint8_t[]){ 2, 1, 3, 0, 4, } },
+    { 4,  105,  625, res_stereo_p3_0, 1,    -2.0,   1.0, (const uint8_t[]){ 2, 1, 3, 0, 4, } },
+    { 2,   68,   81, res_stereo_p4_0, 1,    -4.0,   1.0, (const uint8_t[]){ 4, 3, 5, 2, 6, 1, 7, 0, 8, } },
+    { 2,   81,   81, res_stereo_p5_0, 1,    -4.0,   1.0, (const uint8_t[]){ 4, 3, 5, 2, 6, 1, 7, 0, 8, } },
+    { 2,  289,  289, res_stereo_p6_0, 1,    -8.0,   1.0, (const uint8_t[]){ 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, 0, 16, } },
+    { 4,   81,   81, res_stereo_p7_0, 1,   -11.0,  11.0, (const uint8_t[]){ 1, 0, 2, } },
+    { 2,  121,  121, res_stereo_p7_1, 1,    -5.0,   1.0, (const uint8_t[]){ 5, 4, 6, 3, 7, 2, 8, 1, 9, 0, 10, } },
+    { 2,  169,  169, res_stereo_p8_0, 1,   -30.0,   5.0, (const uint8_t[]){ 6, 5, 7, 4, 8, 3, 9, 2, 10, 1, 11, 0, 12, } },
+    { 2,   25,   25, res_stereo_p8_1, 1,    -2.0,   1.0, (const uint8_t[]){ 2, 1, 3, 0, 4, } },
+    { 2,  169,  169, res_stereo_p9_0, 1, -1530.0, 255.0, (const uint8_t[]){ 6, 5, 7, 4, 8, 3, 9, 2, 10, 1, 11, 0, 12, } },
+    { 2,  225,  225, res_stereo_p9_1, 1,  -119.0,  17.0, (const uint8_t[]){ 7, 6, 8, 5, 9, 4, 10, 3, 11, 2, 12, 1, 13, 0, 14, } },
+    { 2,  289,  289, res_stereo_p9_2, 1,    -8.0,   1.0, (const uint8_t[]){ 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, 0, 16, } },
 };
 
 static const struct {
@@ -598,4 +612,22 @@ static const struct {
       { 3, 2, 5, { -1, 20, 21, 22 } },},
 };
 
+res_setup res_class = {
+    .type = 2,
+    .end = { 208, 1600 },
+    .classifications = 10,
+    .nbooks = 15,
+    .config = res_stereo_config,
+    .books = { { -1, -1, -1, -1, -1, -1, -1, -1, },
+               { -1, -1,  2, -1, -1, -1, -1, -1, },
+               { -1, -1,  3, -1, -1, -1, -1, -1, },
+               { -1, -1,  4, -1, -1, -1, -1, -1, },
+               { -1, -1,  5, -1, -1, -1, -1, -1, },
+               { -1, -1,  6, -1, -1, -1, -1, -1, },
+               { -1, -1,  7, -1, -1, -1, -1, -1, },
+               {  8,  9, -1, -1, -1, -1, -1, -1, },
+               { 10, 11, -1, -1, -1, -1, -1, -1, },
+               { 12, 13, 14, -1, -1, -1, -1, -1, }, },
+};
+
 #endif /* AVCODEC_VORBIS_ENC_DATA_H */
diff --git a/libavcodec/vorbisenc.c b/libavcodec/vorbisenc.c
index fae90c4a30..c2c6e1b7d8 100644
--- a/libavcodec/vorbisenc.c
+++ b/libavcodec/vorbisenc.c
@@ -111,7 +111,7 @@ static int ready_residue(vorbis_enc_residue *rc, vorbis_enc_context *venc)
                 break;
         if (j == 8) // zero
             continue;
-        cb = &venc->codebooks[rc->books[i][j]];
+        cb = &venc->res_books[rc->books[i][j]];
         assert(cb->ndimensions >= 2);
         assert(cb->lookup);
 
@@ -155,7 +155,7 @@ static av_cold int dsp_init(AVCodecContext *avctx, vorbis_enc_context *venc)
     return 0;
 }
 
-static int create_residues(vorbis_enc_context *venc)
+static int create_residues(vorbis_enc_context *venc, res_setup setup)
 {
     int res, ret;
     vorbis_enc_residue *rc;
@@ -167,30 +167,17 @@ static int create_residues(vorbis_enc_context *venc)
 
     for (res = 0; res < venc->nresidues; res++) {
         rc = &venc->residues[res];
-        rc->type            = 2;
+        rc->type            = setup.type;
         rc->begin           = 0;
-        rc->end             = res ? 1600 : 208;
+        rc->end             = setup.end[res];
         rc->partition_size  = res ?   32 :  16;
-        rc->classbook       = res ?   24 :  23;
-        rc->classifications = 10;
+        rc->classbook       = res ?    1 :   0;
+        rc->classifications = setup.classifications;
         rc->books           = av_malloc(sizeof(*rc->books) * rc->classifications);
         if (!rc->books)
             return AVERROR(ENOMEM);
-        {
-            static const int8_t a[10][8] = {
-                { -1, -1, -1, -1, -1, -1, -1, -1, },
-                { -1, -1, 25, -1, -1, -1, -1, -1, },
-                { -1, -1, 26, -1, -1, -1, -1, -1, },
-                { -1, -1, 27, -1, -1, -1, -1, -1, },
-                { -1, -1, 28, -1, -1, -1, -1, -1, },
-                { -1, -1, 29, -1, -1, -1, -1, -1, },
-                { -1, -1, 30, -1, -1, -1, -1, -1, },
-                { 31, 32, -1, -1, -1, -1, -1, -1, },
-                { 33, 34, -1, -1, -1, -1, -1, -1, },
-                { 35, 36, 37, -1, -1, -1, -1, -1, },
-            };
-            memcpy(rc->books, a, sizeof a);
-        }
+
+        memcpy(rc->books, setup.books, sizeof(setup.books));
         if ((ret = ready_residue(rc, venc)) < 0)
             return ret;
     }
@@ -264,49 +251,38 @@ static int create_floors(vorbis_enc_context *venc, AVCodecContext *avctx)
     return 0;
 }
 
-static int create_vorbis_context(vorbis_enc_context *venc,
-                                 AVCodecContext *avctx)
+/**
+ * Copy the codebooks from the hardcoded configurations into the vorbis context
+ */
+static int copy_codebooks(vorbis_enc_codebook *dest,
+                          const codebook_setup *source,
+                          const int num_books)
 {
-    vorbis_enc_mapping *mc;
-    int i, map, book, ret, blocks;
-
-    venc->channels    = avctx->channels;
-    venc->sample_rate = avctx->sample_rate;
-    venc->log2_blocksize[0] = 8;
-    venc->log2_blocksize[1] = 11;
-    venc->blockflags[0] = venc->blockflags[1] = venc->blockflags[2] = 1;
-    venc->transient = -1;
-    venc->num_transient = 1 << (venc->log2_blocksize[1] - venc->log2_blocksize[0]);
-
-    venc->ncodebooks = FF_ARRAY_ELEMS(cvectors);
-    venc->codebooks  = av_malloc(sizeof(vorbis_enc_codebook) * venc->ncodebooks);
-    if (!venc->codebooks)
-        return AVERROR(ENOMEM);
-
-    for (book = 0; book < venc->ncodebooks; book++) {
-        vorbis_enc_codebook *cb = &venc->codebooks[book];
-        int vals;
-        cb->ndimensions = cvectors[book].dim;
-        cb->nentries    = cvectors[book].real_len;
-        cb->min         = cvectors[book].min;
-        cb->delta       = cvectors[book].delta;
-        cb->lookup      = cvectors[book].lookup;
+    int book;
+    for (book = 0; book < num_books; book++) {
+        int ret;
+        vorbis_enc_codebook *cb = &dest[book];
+        cb->ndimensions = source[book].dim;
+        cb->nentries    = source[book].real_len;
+        cb->min         = source[book].min;
+        cb->delta       = source[book].delta;
+        cb->lookup      = source[book].lookup;
         cb->seq_p       = 0;
 
         cb->lens      = av_malloc_array(cb->nentries, sizeof(uint8_t));
         cb->codewords = av_malloc_array(cb->nentries, sizeof(uint32_t));
         if (!cb->lens || !cb->codewords)
             return AVERROR(ENOMEM);
-        memcpy(cb->lens, cvectors[book].clens, cvectors[book].len);
-        memset(cb->lens + cvectors[book].len, 0, cb->nentries - cvectors[book].len);
+        memcpy(cb->lens, source[book].clens, source[book].len);
+        memset(cb->lens + source[book].len, 0, cb->nentries - source[book].len);
 
         if (cb->lookup) {
-            vals = cb_lookup_vals(cb->lookup, cb->ndimensions, cb->nentries);
+            int i, vals = cb_lookup_vals(cb->lookup, cb->ndimensions, cb->nentries);
             cb->quantlist = av_malloc_array(vals, sizeof(int));
             if (!cb->quantlist)
                 return AVERROR(ENOMEM);
             for (i = 0; i < vals; i++)
-                cb->quantlist[i] = cvectors[book].quant[i];
+                cb->quantlist[i] = source[book].quant[i];
         } else {
             cb->quantlist = NULL;
         }
@@ -314,10 +290,41 @@ static int create_vorbis_context(vorbis_enc_context *venc,
             return ret;
     }
 
+    return 0;
+}
+
+static int create_vorbis_context(vorbis_enc_context *venc,
+                                 AVCodecContext *avctx)
+{
+    vorbis_enc_mapping *mc;
+    int i, map, ret, blocks;
+
+    venc->channels    = avctx->channels;
+    venc->sample_rate = avctx->sample_rate;
+    venc->log2_blocksize[0] = 8;
+    venc->log2_blocksize[1] = 11;
+    venc->blockflags[0] = venc->blockflags[1] = venc->blockflags[2] = 1;
+    venc->transient = -1;
+    venc->num_transient = 1 << (venc->log2_blocksize[1] - venc->log2_blocksize[0]);
+
+    // Setup and configure our floors
+    venc->nfloor_books = FF_ARRAY_ELEMS(floor_config);
+    venc->floor_books = av_malloc(sizeof(vorbis_enc_codebook) * venc->nfloor_books);
+    if (!venc->floor_books)
+        return AVERROR(ENOMEM);
+
+    copy_codebooks(venc->floor_books, floor_config, venc->nfloor_books);
     if ((ret = create_floors(venc, avctx)) < 0)
         return ret;
 
-    if ((ret = create_residues(venc)) < 0)
+    // Setup and configure our residues
+    venc->nres_books = res_class.nbooks;
+    venc->res_books = av_malloc(sizeof(vorbis_enc_codebook) * venc->nres_books);
+    if (!venc->res_books)
+        return AVERROR(ENOMEM);
+
+    copy_codebooks(venc->res_books, res_class.config, venc->nres_books);
+    if ((ret = create_residues(venc, res_class)) < 0)
         return ret;
 
     venc->nmappings = 2;
@@ -497,7 +504,8 @@ static void put_floor_header(PutBitContext *pb, vorbis_enc_floor *fc)
         put_bits(pb, fc->rangebits, fc->list[i].x);
 }
 
-static void put_residue_header(PutBitContext *pb, vorbis_enc_residue *rc)
+static void put_residue_header(PutBitContext *pb, vorbis_enc_residue *rc,
+                               const int book_offset)
 {
     int i;
 
@@ -507,7 +515,7 @@ static void put_residue_header(PutBitContext *pb, vorbis_enc_residue *rc)
     put_bits(pb, 24, rc->end);
     put_bits(pb, 24, rc->partition_size - 1);
     put_bits(pb, 6, rc->classifications - 1);
-    put_bits(pb, 8, rc->classbook);
+    put_bits(pb, 8, book_offset + rc->classbook);
 
     for (i = 0; i < rc->classifications; i++) {
         int j, tmp = 0;
@@ -525,7 +533,7 @@ static void put_residue_header(PutBitContext *pb, vorbis_enc_residue *rc)
         int j;
         for (j = 0; j < 8; j++)
             if (rc->books[i][j] != -1)
-                put_bits(pb, 8, rc->books[i][j]);
+                put_bits(pb, 8, book_offset + rc->books[i][j]);
     }
 }
 
@@ -580,9 +588,12 @@ static int put_main_header(vorbis_enc_context *venc, uint8_t **out)
         put_bits(&pb, 8, "vorbis"[i]);
 
     // codebooks
-    put_bits(&pb, 8, venc->ncodebooks - 1);
-    for (i = 0; i < venc->ncodebooks; i++)
-        put_codebook_header(&pb, &venc->codebooks[i]);
+    put_bits(&pb, 8, venc->nfloor_books + venc->nres_books - 1);
+    for (i = 0; i < venc->nfloor_books; i++)
+        put_codebook_header(&pb, &venc->floor_books[i]);
+
+    for (i = 0; i < venc->nres_books; i++)
+        put_codebook_header(&pb, &venc->res_books[i]);
 
     // time domain, reserved, zero
     put_bits(&pb,  6, 0);
@@ -596,7 +607,7 @@ static int put_main_header(vorbis_enc_context *venc, uint8_t **out)
     // residues
     put_bits(&pb, 6, venc->nresidues - 1);
     for (i = 0; i < venc->nresidues; i++)
-        put_residue_header(&pb, &venc->residues[i]);
+        put_residue_header(&pb, &venc->residues[i], venc->nfloor_books);
 
     // mappings
     put_bits(&pb, 6, venc->nmappings - 1);
@@ -759,14 +770,14 @@ static int floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc,
         vorbis_enc_floor_class * c = &fc->classes[fc->partition_to_class[i]];
         int k, cval = 0, csub = 1<<c->subclass;
         if (c->subclass) {
-            vorbis_enc_codebook * book = &venc->codebooks[c->masterbook];
+            vorbis_enc_codebook * book = &venc->floor_books[c->masterbook];
             int cshift = 0;
             for (k = 0; k < c->dim; k++) {
                 int l;
                 for (l = 0; l < csub; l++) {
                     int maxval = 1;
                     if (c->books[l] != -1)
-                        maxval = venc->codebooks[c->books[l]].nentries;
+                        maxval = venc->floor_books[c->books[l]].nentries;
                     // coded could be -1, but this still works, cause that is 0
                     if (coded[counter + k] < maxval)
                         break;
@@ -786,7 +797,7 @@ static int floor_encode(vorbis_enc_context *venc, vorbis_enc_floor *fc,
                 continue;
             if (entry == -1)
                 entry = 0;
-            if (put_codeword(pb, &venc->codebooks[book], entry))
+            if (put_codeword(pb, &venc->floor_books[book], entry))
                 return AVERROR(EINVAL);
         }
     }
@@ -829,7 +840,7 @@ static int residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
     int partitions = (rc->end - rc->begin) / psize;
     int channels   = (rc->type == 2) ? 1 : real_ch;
     int classes[MAX_CHANNELS][NUM_RESIDUE_PARTITIONS];
-    int classwords = venc->codebooks[rc->classbook].ndimensions;
+    int classwords = venc->res_books[rc->classbook].ndimensions;
 
     av_assert0(rc->type == 2);
     av_assert0(real_ch == 2);
@@ -852,7 +863,7 @@ static int residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
         while (p < partitions) {
             if (pass == 0)
                 for (j = 0; j < channels; j++) {
-                    vorbis_enc_codebook * book = &venc->codebooks[rc->classbook];
+                    vorbis_enc_codebook * book = &venc->res_books[rc->classbook];
                     int entry = classes[j][p];
                     for (i = 1; i < classwords; i++) {
                         entry *= rc->classifications;
@@ -865,7 +876,7 @@ static int residue_encode(vorbis_enc_context *venc, vorbis_enc_residue *rc,
             for (i = 0; i < classwords && p < partitions; i++, p++) {
                 for (j = 0; j < channels; j++) {
                     int nbook = rc->books[classes[j][p]][pass];
-                    vorbis_enc_codebook * book = &venc->codebooks[nbook];
+                    vorbis_enc_codebook * book = &venc->res_books[nbook];
                     float *buf = coeffs + samples*j + rc->begin + p*psize;
                     if (nbook == -1)
                         continue;
@@ -1186,15 +1197,25 @@ static av_cold int vorbis_encode_close(AVCodecContext *avctx)
     vorbis_enc_context *venc = avctx->priv_data;
     int i;
 
-    if (venc->codebooks)
-        for (i = 0; i < venc->ncodebooks; i++) {
-            av_freep(&venc->codebooks[i].lens);
-            av_freep(&venc->codebooks[i].codewords);
-            av_freep(&venc->codebooks[i].quantlist);
-            av_freep(&venc->codebooks[i].dimensions);
-            av_freep(&venc->codebooks[i].pow2);
+    if (venc->floor_books)
+        for (i = 0; i < venc->nfloor_books; i++) {
+            av_freep(&venc->floor_books[i].lens);
+            av_freep(&venc->floor_books[i].codewords);
+            av_freep(&venc->floor_books[i].quantlist);
+            av_freep(&venc->floor_books[i].dimensions);
+            av_freep(&venc->floor_books[i].pow2);
+        }
+    av_freep(&venc->floor_books);
+
+    if (venc->res_books)
+        for (i = 0; i < venc->nres_books; i++) {
+            av_freep(&venc->res_books[i].lens);
+            av_freep(&venc->res_books[i].codewords);
+            av_freep(&venc->res_books[i].quantlist);
+            av_freep(&venc->res_books[i].dimensions);
+            av_freep(&venc->res_books[i].pow2);
         }
-    av_freep(&venc->codebooks);
+    av_freep(&venc->res_books);
 
     if (venc->floors)
         for (i = 0; i < venc->nfloors; i++) {
