diff mbox series

[FFmpeg-devel,4/5] avcodec/bsf/vvc_mp4toannexb: Factor creating new extradata out

Message ID AS8P250MB0744271220341ACA1ADC40B88F522@AS8P250MB0744.EURP250.PROD.OUTLOOK.COM
State New
Headers show
Series [FFmpeg-devel,1/5] avcodec/bsf/(hevc|vvc)_mp4toannexb: Ensure extradata_size < INT_MAX | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Andreas Rheinhardt Feb. 18, 2024, 2:44 a.m. UTC 
This is in preparation for the next commit.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>
---
 libavcodec/bsf/vvc_mp4toannexb.c | 116 ++++++++++++++++++-------------
 1 file changed, 67 insertions(+), 49 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/bsf/vvc_mp4toannexb.c b/libavcodec/bsf/vvc_mp4toannexb.c
index 1b851f3223..bfb0338116 100644
--- a/libavcodec/bsf/vvc_mp4toannexb.c
+++ b/libavcodec/bsf/vvc_mp4toannexb.c
@@ -37,16 +37,77 @@  typedef struct VVCBSFContext {
     int extradata_parsed;
 } VVCBSFContext;
 
+static int vvc_extradata_to_annexb_internal(void *logctx, GetByteContext *gb,
+                                            uint8_t **new_extradatap,
+                                            size_t *new_extradata_sizep)
+{
+    int num_arrays = bytestream2_get_byte(gb);
+    uint8_t *new_extradata = NULL;
+    size_t new_extradata_size = 0;
+    int ret;
+
+    for (int i = 0; i < num_arrays; i++) {
+        int cnt;
+        int type = bytestream2_get_byte(gb) & 0x1f;
+
+        if (type == VVC_OPI_NUT || type == VVC_DCI_NUT)
+            cnt = 1;
+        else
+            cnt = bytestream2_get_be16(gb);
+
+        av_log(logctx, AV_LOG_DEBUG, "nalu_type %d cnt %d\n", type, cnt);
+
+        if (!(type == VVC_OPI_NUT || type == VVC_DCI_NUT ||
+              type == VVC_VPS_NUT || type == VVC_SPS_NUT || type == VVC_PPS_NUT
+              || type == VVC_PREFIX_SEI_NUT || type == VVC_SUFFIX_SEI_NUT)) {
+            av_log(logctx, AV_LOG_ERROR,
+                   "Invalid NAL unit type in extradata: %d\n", type);
+            ret = AVERROR_INVALIDDATA;
+            goto fail;
+        }
+
+        for (int j = 0; j < cnt; j++) {
+            const int nalu_len = bytestream2_get_be16(gb);
+
+            if (!nalu_len ||
+                nalu_len > bytestream2_get_bytes_left(gb) ||
+                4 + nalu_len > FFMIN(INT_MAX, SIZE_MAX) - AV_INPUT_BUFFER_PADDING_SIZE - new_extradata_size) {
+                ret = AVERROR_INVALIDDATA;
+                goto fail;
+            }
+            ret = av_reallocp(&new_extradata, new_extradata_size + nalu_len + 4
+                              + AV_INPUT_BUFFER_PADDING_SIZE);
+            if (ret < 0)
+                goto fail;
+
+            AV_WB32(new_extradata + new_extradata_size, 1); // add the startcode
+            bytestream2_get_buffer(gb, new_extradata + new_extradata_size + 4,
+                                   nalu_len);
+            new_extradata_size += 4 + nalu_len;
+            memset(new_extradata + new_extradata_size, 0,
+                   AV_INPUT_BUFFER_PADDING_SIZE);
+        }
+    }
+
+    *new_extradatap      = new_extradata;
+    *new_extradata_sizep = new_extradata_size;
+
+    return 0;
+fail:
+    av_freep(&new_extradata);
+    return ret;
+}
+
 static int vvc_extradata_to_annexb(AVBSFContext *ctx)
 {
     GetByteContext gb;
-    int length_size, num_arrays, i, j;
+    int length_size, i, j;
     int ret = 0;
     int temp = 0;
     int ptl_present;
 
     uint8_t *new_extradata = NULL;
-    size_t new_extradata_size = 0;
+    size_t new_extradata_size;
 
     int max_picture_width = 0;
     int max_picture_height = 0;
@@ -132,50 +193,10 @@  static int vvc_extradata_to_annexb(AVBSFContext *ctx)
                max_picture_width, max_picture_height, avg_frame_rate);
     }
 
-    num_arrays = bytestream2_get_byte(&gb);
-
-    for (i = 0; i < num_arrays; i++) {
-        int cnt;
-        int type = bytestream2_get_byte(&gb) & 0x1f;
-
-        if (type == VVC_OPI_NUT || type == VVC_DCI_NUT)
-            cnt = 1;
-        else
-            cnt = bytestream2_get_be16(&gb);
-
-        av_log(ctx, AV_LOG_DEBUG, "nalu_type %d cnt %d\n", type, cnt);
-
-        if (!(type == VVC_OPI_NUT || type == VVC_DCI_NUT ||
-              type == VVC_VPS_NUT || type == VVC_SPS_NUT || type == VVC_PPS_NUT
-              || type == VVC_PREFIX_SEI_NUT || type == VVC_SUFFIX_SEI_NUT)) {
-            av_log(ctx, AV_LOG_ERROR,
-                   "Invalid NAL unit type in extradata: %d\n", type);
-            ret = AVERROR_INVALIDDATA;
-            goto fail;
-        }
-
-        for (j = 0; j < cnt; j++) {
-            const int nalu_len = bytestream2_get_be16(&gb);
-
-            if (!nalu_len ||
-                nalu_len > bytestream2_get_bytes_left(&gb) ||
-                4 + nalu_len > FFMIN(INT_MAX, SIZE_MAX) - AV_INPUT_BUFFER_PADDING_SIZE - new_extradata_size) {
-                ret = AVERROR_INVALIDDATA;
-                goto fail;
-            }
-            ret = av_reallocp(&new_extradata, new_extradata_size + nalu_len + 4
-                              + AV_INPUT_BUFFER_PADDING_SIZE);
-            if (ret < 0)
-                goto fail;
-
-            AV_WB32(new_extradata + new_extradata_size, 1); // add the startcode
-            bytestream2_get_buffer(&gb, new_extradata + new_extradata_size + 4,
-                                   nalu_len);
-            new_extradata_size += 4 + nalu_len;
-            memset(new_extradata + new_extradata_size, 0,
-                   AV_INPUT_BUFFER_PADDING_SIZE);
-        }
-    }
+    ret = vvc_extradata_to_annexb_internal(ctx, &gb, &new_extradata,
+                                           &new_extradata_size);
+    if (ret < 0)
+        return ret;
 
     av_freep(&ctx->par_out->extradata);
     ctx->par_out->extradata = new_extradata;
@@ -185,9 +206,6 @@  static int vvc_extradata_to_annexb(AVBSFContext *ctx)
         av_log(ctx, AV_LOG_WARNING, "No parameter sets in the extradata\n");
 
     return length_size;
-  fail:
-    av_freep(&new_extradata);
-    return ret;
 }
 
 static int vvc_mp4toannexb_init(AVBSFContext *ctx)