From f3513f992e0b5595f2644257b92fdea6189592de Mon Sep 17 00:00:00 2001
From: Ulf Zibis <Ulf.Zibis@CoSoCo.de>
Date: 07.03.2019, 00:34:51

Correct usage of linesize and width in 16 bit depth routines.

diff --git a/.gitignore b/.gitignore
index 0e57cb0..7819c84 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,5 @@
 /lcov/
 /src
 /mapfile
+/nbproject
+/debug
diff --git a/libavfilter/vf_fillborders.c b/libavfilter/vf_fillborders.c
index 1344587..019cdff 100644
--- a/libavfilter/vf_fillborders.c
+++ b/libavfilter/vf_fillborders.c
@@ -89,25 +89,27 @@
     for (p = 0; p < s->nb_planes; p++) {
         uint8_t *ptr = frame->data[p];
         int linesize = frame->linesize[p];
+        int nb_leftbytes = s->borders[p].left * linesize / s->planewidth[p];
+        int nb_rightbytes = s->borders[p].right * linesize / s->planewidth[p];
 
         for (y = s->borders[p].top; y < s->planeheight[p] - s->borders[p].bottom; y++) {
             memset(ptr + y * linesize,
-                   *(ptr + y * linesize + s->borders[p].left),
-                   s->borders[p].left);
-            memset(ptr + y * linesize + s->planewidth[p] - s->borders[p].right,
-                   *(ptr + y * linesize + s->planewidth[p] - s->borders[p].right - 1),
-                   s->borders[p].right);
+                   *(ptr + y * linesize + nb_leftbytes),
+                   nb_leftbytes);
+            memset(ptr + y * linesize + linesize - nb_rightbytes,
+                   *(ptr + y * linesize + linesize - nb_rightbytes - 1),
+                   nb_rightbytes);
         }
 
         for (y = 0; y < s->borders[p].top; y++) {
             memcpy(ptr + y * linesize,
-                   ptr + s->borders[p].top * linesize, s->planewidth[p]);
+                   ptr + s->borders[p].top * linesize, linesize);
         }
 
         for (y = s->planeheight[p] - s->borders[p].bottom; y < s->planeheight[p]; y++) {
             memcpy(ptr + y * linesize,
                    ptr + (s->planeheight[p] - s->borders[p].bottom - 1) * linesize,
-                   s->planewidth[p]);
+                   linesize);
         }
     }
 }
@@ -118,28 +120,28 @@
 
     for (p = 0; p < s->nb_planes; p++) {
         uint16_t *ptr = (uint16_t *)frame->data[p];
-        int linesize = frame->linesize[p] / 2;
+        int linesize = frame->linesize[p];
 
         for (y = s->borders[p].top; y < s->planeheight[p] - s->borders[p].bottom; y++) {
             for (x = 0; x < s->borders[p].left; x++) {
-                ptr[y * linesize + x] =  *(ptr + y * linesize + s->borders[p].left);
+                ptr[y * s->planewidth[p] + x] =  *(ptr + y * s->planewidth[p] + s->borders[p].left);
             }
 
             for (x = 0; x < s->borders[p].right; x++) {
-                ptr[y * linesize + s->planewidth[p] - s->borders[p].right + x] =
-                   *(ptr + y * linesize + s->planewidth[p] - s->borders[p].right - 1);
+                ptr[y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right + x] =
+                   *(ptr + y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right - 1);
             }
         }
 
         for (y = 0; y < s->borders[p].top; y++) {
             memcpy(ptr + y * linesize,
-                   ptr + s->borders[p].top * linesize, s->planewidth[p] * 2);
+                   ptr + s->borders[p].top * linesize, linesize);
         }
 
         for (y = s->planeheight[p] - s->borders[p].bottom; y < s->planeheight[p]; y++) {
             memcpy(ptr + y * linesize,
                    ptr + (s->planeheight[p] - s->borders[p].bottom - 1) * linesize,
-                   s->planewidth[p] * 2);
+                   linesize);
         }
     }
 }
@@ -154,25 +156,25 @@
 
         for (y = s->borders[p].top; y < s->planeheight[p] - s->borders[p].bottom; y++) {
             for (x = 0; x < s->borders[p].left; x++) {
-                ptr[y * linesize + x] = ptr[y * linesize + s->borders[p].left * 2 - 1 - x];
+                ptr[y * s->planewidth[p] + x] = ptr[y * s->planewidth[p] + s->borders[p].left * 2 - 1 - x];
             }
 
             for (x = 0; x < s->borders[p].right; x++) {
-                ptr[y * linesize + s->planewidth[p] - s->borders[p].right + x] =
-                    ptr[y * linesize + s->planewidth[p] - s->borders[p].right - 1 - x];
+                ptr[y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right + x] =
+                    ptr[y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right - 1 - x];
             }
         }
 
         for (y = 0; y < s->borders[p].top; y++) {
             memcpy(ptr + y * linesize,
                    ptr + (s->borders[p].top * 2 - 1 - y) * linesize,
-                   s->planewidth[p]);
+                   linesize);
         }
 
         for (y = 0; y < s->borders[p].bottom; y++) {
             memcpy(ptr + (s->planeheight[p] - s->borders[p].bottom + y) * linesize,
                    ptr + (s->planeheight[p] - s->borders[p].bottom - 1 - y) * linesize,
-                   s->planewidth[p]);
+                   linesize);
         }
     }
 }
@@ -183,29 +185,29 @@
 
     for (p = 0; p < s->nb_planes; p++) {
         uint16_t *ptr = (uint16_t *)frame->data[p];
-        int linesize = frame->linesize[p] / 2;
+        int linesize = frame->linesize[p];
 
         for (y = s->borders[p].top; y < s->planeheight[p] - s->borders[p].bottom; y++) {
             for (x = 0; x < s->borders[p].left; x++) {
-                ptr[y * linesize + x] = ptr[y * linesize + s->borders[p].left * 2 - 1 - x];
+                ptr[y * s->planewidth[p] + x] = ptr[y * s->planewidth[p] + s->borders[p].left * 2 - 1 - x];
             }
 
             for (x = 0; x < s->borders[p].right; x++) {
-                ptr[y * linesize + s->planewidth[p] - s->borders[p].right + x] =
-                    ptr[y * linesize + s->planewidth[p] - s->borders[p].right - 1 - x];
+                ptr[y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right + x] =
+                    ptr[y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right - 1 - x];
             }
         }
 
         for (y = 0; y < s->borders[p].top; y++) {
             memcpy(ptr + y * linesize,
                    ptr + (s->borders[p].top * 2 - 1 - y) * linesize,
-                   s->planewidth[p] * 2);
+                   linesize);
         }
 
         for (y = 0; y < s->borders[p].bottom; y++) {
             memcpy(ptr + (s->planeheight[p] - s->borders[p].bottom + y) * linesize,
                    ptr + (s->planeheight[p] - s->borders[p].bottom - 1 - y) * linesize,
-                   s->planewidth[p] * 2);
+                   linesize);
         }
     }
 }
@@ -216,21 +218,23 @@
 
     for (p = 0; p < s->nb_planes; p++) {
         uint8_t *ptr = frame->data[p];
-        uint8_t fill = s->fill[p];
         int linesize = frame->linesize[p];
+        int nb_leftbytes = s->borders[p].left * linesize / s->planewidth[p];
+        int nb_rightbytes = s->borders[p].right * linesize / s->planewidth[p];
+        uint8_t fill = s->fill[p];
 
         for (y = s->borders[p].top; y < s->planeheight[p] - s->borders[p].bottom; y++) {
-            memset(ptr + y * linesize, fill, s->borders[p].left);
-            memset(ptr + y * linesize + s->planewidth[p] - s->borders[p].right, fill,
-                   s->borders[p].right);
+            memset(ptr + y * linesize, fill, nb_leftbytes);
+            memset(ptr + y * linesize + linesize - nb_rightbytes, fill,
+                   nb_rightbytes);
         }
 
         for (y = 0; y < s->borders[p].top; y++) {
-            memset(ptr + y * linesize, fill, s->planewidth[p]);
+            memset(ptr + y * linesize, fill, linesize);
         }
 
         for (y = s->planeheight[p] - s->borders[p].bottom; y < s->planeheight[p]; y++) {
-            memset(ptr + y * linesize, fill, s->planewidth[p]);
+            memset(ptr + y * linesize, fill, linesize);
         }
     }
 }
@@ -242,27 +246,26 @@
     for (p = 0; p < s->nb_planes; p++) {
         uint16_t *ptr = (uint16_t *)frame->data[p];
         uint16_t fill = s->fill[p] << (s->depth - 8);
-        int linesize = frame->linesize[p] / 2;
 
         for (y = s->borders[p].top; y < s->planeheight[p] - s->borders[p].bottom; y++) {
             for (x = 0; x < s->borders[p].left; x++) {
-                ptr[y * linesize + x] = fill;
+                ptr[y * s->planewidth[p] + x] = fill;
             }
 
             for (x = 0; x < s->borders[p].right; x++) {
-                ptr[y * linesize + s->planewidth[p] - s->borders[p].right + x] = fill;
+                ptr[y * s->planewidth[p] + s->planewidth[p] - s->borders[p].right + x] = fill;
             }
         }
 
         for (y = 0; y < s->borders[p].top; y++) {
             for (x = 0; x < s->planewidth[p]; x++) {
-                ptr[y * linesize + x] = fill;
+                ptr[y * s->planewidth[p] + x] = fill;
             }
         }
 
         for (y = s->planeheight[p] - s->borders[p].bottom; y < s->planeheight[p]; y++) {
             for (x = 0; x < s->planewidth[p]; x++) {
-                ptr[y * linesize + x] = fill;
+                ptr[y * s->planewidth[p] + x] = fill;
             }
         }
     }
@@ -281,44 +284,38 @@
 {
     AVFilterContext *ctx = inlink->dst;
     FillBordersContext *s = ctx->priv;
+    if (inlink->w < s->left + s->right ||
+            inlink->w <= s->left ||
+            inlink->w <= s->right ||
+            inlink->h < s->top + s->bottom ||
+            inlink->h <= s->top ||
+            inlink->h <= s->bottom ||
+            inlink->w < s->left * 2 ||
+            inlink->w < s->right * 2 ||
+            inlink->h < s->top * 2 ||
+            inlink->h < s->bottom * 2) {
+        av_log(ctx, AV_LOG_ERROR, "Borders are bigger than input frame size.\n");
+        return AVERROR(EINVAL);
+        }
+
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
 
     s->nb_planes = desc->nb_components;
     s->depth = desc->comp[0].depth;
 
-    s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
     s->planeheight[0] = s->planeheight[3] = inlink->h;
-    s->planewidth[1]  = s->planewidth[2]  = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
+    s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
     s->planewidth[0]  = s->planewidth[3]  = inlink->w;
+    s->planewidth[1]  = s->planewidth[2]  = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
 
-    s->borders[0].left   = s->borders[3].left = s->left;
-    s->borders[0].right  = s->borders[3].right = s->right;
-    s->borders[0].top    = s->borders[3].top = s->top;
+    s->borders[0].left   = s->borders[3].left   = s->left;
+    s->borders[1].left   = s->borders[2].left   = s->left >> desc->log2_chroma_w;
+    s->borders[0].right  = s->borders[3].right  = s->right;
+    s->borders[1].right  = s->borders[2].right  = s->right >> desc->log2_chroma_w;
+    s->borders[0].top    = s->borders[3].top    = s->top;
+    s->borders[1].top    = s->borders[2].top    = s->top >> desc->log2_chroma_h;
     s->borders[0].bottom = s->borders[3].bottom = s->bottom;
-
-    s->borders[1].left   = s->left >> desc->log2_chroma_w;
-    s->borders[1].right  = s->right >> desc->log2_chroma_w;
-    s->borders[1].top    = s->top >> desc->log2_chroma_h;
-    s->borders[1].bottom = s->bottom >> desc->log2_chroma_h;
-
-    s->borders[2].left   = s->left >> desc->log2_chroma_w;
-    s->borders[2].right  = s->right >> desc->log2_chroma_w;
-    s->borders[2].top    = s->top >> desc->log2_chroma_h;
-    s->borders[2].bottom = s->bottom >> desc->log2_chroma_h;
-
-    if (inlink->w < s->left + s->right ||
-        inlink->w <= s->left ||
-        inlink->w <= s->right ||
-        inlink->h < s->top + s->bottom ||
-        inlink->h <= s->top ||
-        inlink->h <= s->bottom ||
-        inlink->w < s->left * 2 ||
-        inlink->w < s->right * 2 ||
-        inlink->h < s->top * 2 ||
-        inlink->h < s->bottom * 2) {
-        av_log(ctx, AV_LOG_ERROR, "Borders are bigger than input frame size.\n");
-        return AVERROR(EINVAL);
-    }
+    s->borders[1].bottom = s->borders[2].bottom = s->bottom >> desc->log2_chroma_h;
 
     switch (s->mode) {
     case FM_SMEAR:  s->fillborders = s->depth <= 8 ? smear_borders8  : smear_borders16;  break;
