| Message ID | 20210205120811.1618152-1-andreas.rheinhardt@gmail.com |
|---|---|
| State | Accepted |
| Commit | 8d21eccd267acfcde3d35bbbf6621d6c3282e1ea |
| Headers | show |
| Series | [FFmpeg-devel] avcodec/g722enc: Validate parameters before using them | expand |
| Context | Check | Description |
|---|---|---|
| andriy/x86_make | success | Make finished |
| andriy/x86_make_fate | success | Make fate finished |
| andriy/PPC64_make | success | Make finished |
| andriy/PPC64_make_fate | success | Make fate finished |
Andreas Rheinhardt: > In case trellis is outside of 0..23, an invalid shift and/or a signed > integer overflow happens; furthermore, it can lead to the request to > allocate nonsense amounts of memory. So validate first. > > Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com> > --- > libavcodec/g722enc.c | 25 ++++++++++++------------- > 1 file changed, 12 insertions(+), 13 deletions(-) > > diff --git a/libavcodec/g722enc.c b/libavcodec/g722enc.c > index 9357f170fe..9e2ebf67c5 100644 > --- a/libavcodec/g722enc.c > +++ b/libavcodec/g722enc.c > @@ -64,19 +64,6 @@ static av_cold int g722_encode_init(AVCodecContext * avctx) > c->band[1].scale_factor = 2; > c->prev_samples_pos = 22; > > - if (avctx->trellis) { > - int frontier = 1 << avctx->trellis; > - int max_paths = frontier * FREEZE_INTERVAL; > - int i; > - for (i = 0; i < 2; i++) { > - c->paths[i] = av_mallocz_array(max_paths, sizeof(**c->paths)); > - c->node_buf[i] = av_mallocz_array(frontier, 2 * sizeof(**c->node_buf)); > - c->nodep_buf[i] = av_mallocz_array(frontier, 2 * sizeof(**c->nodep_buf)); > - if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i]) > - return AVERROR(ENOMEM); > - } > - } > - > if (avctx->frame_size) { > /* validate frame size */ > if (avctx->frame_size & 1 || avctx->frame_size > MAX_FRAME_SIZE) { > @@ -110,6 +97,18 @@ static av_cold int g722_encode_init(AVCodecContext * avctx) > avctx->trellis); > avctx->trellis = new_trellis; > } > + if (avctx->trellis) { > + int frontier = 1 << avctx->trellis; > + int max_paths = frontier * FREEZE_INTERVAL; > + > + for (int i = 0; i < 2; i++) { > + c->paths[i] = av_calloc(max_paths, sizeof(**c->paths)); > + c->node_buf[i] = av_calloc(frontier, 2 * sizeof(**c->node_buf)); > + c->nodep_buf[i] = av_calloc(frontier, 2 * sizeof(**c->nodep_buf)); > + if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i]) > + return AVERROR(ENOMEM); > + } > + } > } > > ff_g722dsp_init(&c->dsp); > Will apply later today unless there are objections. - Andreas
diff --git a/libavcodec/g722enc.c b/libavcodec/g722enc.c index 9357f170fe..9e2ebf67c5 100644 --- a/libavcodec/g722enc.c +++ b/libavcodec/g722enc.c @@ -64,19 +64,6 @@ static av_cold int g722_encode_init(AVCodecContext * avctx) c->band[1].scale_factor = 2; c->prev_samples_pos = 22; - if (avctx->trellis) { - int frontier = 1 << avctx->trellis; - int max_paths = frontier * FREEZE_INTERVAL; - int i; - for (i = 0; i < 2; i++) { - c->paths[i] = av_mallocz_array(max_paths, sizeof(**c->paths)); - c->node_buf[i] = av_mallocz_array(frontier, 2 * sizeof(**c->node_buf)); - c->nodep_buf[i] = av_mallocz_array(frontier, 2 * sizeof(**c->nodep_buf)); - if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i]) - return AVERROR(ENOMEM); - } - } - if (avctx->frame_size) { /* validate frame size */ if (avctx->frame_size & 1 || avctx->frame_size > MAX_FRAME_SIZE) { @@ -110,6 +97,18 @@ static av_cold int g722_encode_init(AVCodecContext * avctx) avctx->trellis); avctx->trellis = new_trellis; } + if (avctx->trellis) { + int frontier = 1 << avctx->trellis; + int max_paths = frontier * FREEZE_INTERVAL; + + for (int i = 0; i < 2; i++) { + c->paths[i] = av_calloc(max_paths, sizeof(**c->paths)); + c->node_buf[i] = av_calloc(frontier, 2 * sizeof(**c->node_buf)); + c->nodep_buf[i] = av_calloc(frontier, 2 * sizeof(**c->nodep_buf)); + if (!c->paths[i] || !c->node_buf[i] || !c->nodep_buf[i]) + return AVERROR(ENOMEM); + } + } } ff_g722dsp_init(&c->dsp);
In case trellis is outside of 0..23, an invalid shift and/or a signed integer overflow happens; furthermore, it can lead to the request to allocate nonsense amounts of memory. So validate first. Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com> --- libavcodec/g722enc.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-)