diff mbox series

[FFmpeg-devel,v2,02/31] lavu/fifo: disallow overly large fifo sizes

Message ID AM7PR03MB6660ACB6F6E2CE3F3002049A8F5E9@AM7PR03MB6660.eurprd03.prod.outlook.com
State Accepted
Commit 5939c8d3616a559fb5994e570c96e1a0ecad528b
Headers show
Series New FIFO API | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_ppc success Make finished
andriy/make_fate_ppc success Make fate finished
andriy/make_aarch64_jetson success Make finished
andriy/make_fate_aarch64_jetson success Make fate finished

Commit Message

Andreas Rheinhardt Jan. 24, 2022, 2:45 p.m. UTC 
From: Anton Khirnov <anton@khirnov.net>

The API currently allows creating FIFOs up to
- UINT_MAX: av_fifo_alloc(), av_fifo_realloc(), av_fifo_grow()
- SIZE_MAX: av_fifo_alloc_array()
However the usable limit is determined by
- rndx/wndx being uint32_t
- av_fifo_[size,space] returning int
so no FIFO should be larger than the smallest of
- INT_MAX
- UINT32_MAX
- SIZE_MAX
(which should be INT_MAX an all commonly used platforms).
Return an error on trying to allocate FIFOs larger than this limit.
---
 libavutil/fifo.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)
diff mbox series

Patch

diff --git a/libavutil/fifo.c b/libavutil/fifo.c
index e1f2175530..55621f0dca 100644
--- a/libavutil/fifo.c
+++ b/libavutil/fifo.c
@@ -20,14 +20,23 @@ 
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include <stdint.h>
+
 #include "avassert.h"
 #include "common.h"
 #include "fifo.h"
 
+#define OLD_FIFO_SIZE_MAX (size_t)FFMIN3(INT_MAX, UINT32_MAX, SIZE_MAX)
+
 AVFifoBuffer *av_fifo_alloc_array(size_t nmemb, size_t size)
 {
     AVFifoBuffer *f;
-    void *buffer = av_realloc_array(NULL, nmemb, size);
+    void *buffer;
+
+    if (nmemb > OLD_FIFO_SIZE_MAX / size)
+        return NULL;
+
+    buffer = av_realloc_array(NULL, nmemb, size);
     if (!buffer)
         return NULL;
     f = av_mallocz(sizeof(AVFifoBuffer));
@@ -82,6 +91,9 @@  int av_fifo_realloc2(AVFifoBuffer *f, unsigned int new_size)
 {
     unsigned int old_size = f->end - f->buffer;
 
+    if (new_size > OLD_FIFO_SIZE_MAX)
+        return AVERROR(EINVAL);
+
     if (old_size < new_size) {
         size_t offset_r = f->rptr - f->buffer;
         size_t offset_w = f->wptr - f->buffer;