@@ -1144,5 +1144,5 @@ void ff_sws_slice_worker(void *priv, int jobnr, int threadnr,
#define MAX_LINES_AHEAD 4
//shuffle filter and filterPos for hyScale and hcScale filters in avx2
-void ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW);
+int ff_shuffle_filter_coefficients(SwsContext *c, int* filterPos, int filterSize, int16_t *filter, int dstW);
#endif /* SWSCALE_SWSCALE_INTERNAL_H */
@@ -278,7 +278,7 @@ static const FormatEntry format_entries[] = {
[AV_PIX_FMT_P416LE] = { 1, 1 },
};
-void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSize, int16_t *filter, int dstW){
+int ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSize, int16_t *filter, int dstW){
#if ARCH_X86_64
int i, j, k, l;
int cpu_flags = av_get_cpu_flags();
@@ -292,6 +292,9 @@ void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSiz
}
if (filterSize > 4){
int16_t *tmp2 = av_malloc(dstW * filterSize * 2);
+ if (!tmp2)
+ return AVERROR(ENOMEM);
+
memcpy(tmp2, filter, dstW * filterSize * 2);
for (i = 0; i < dstW; i += 16){//pixel
for (k = 0; k < filterSize / 4; ++k){//fcoeff
@@ -311,6 +314,8 @@ void ff_shuffle_filter_coefficients(SwsContext *c, int *filterPos, int filterSiz
}
}
#endif
+
+ return 0;
}
int sws_isSupportedInput(enum AVPixelFormat pix_fmt)
@@ -1836,7 +1841,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
get_local_pos(c, 0, 0, 0),
get_local_pos(c, 0, 0, 0))) < 0)
goto fail;
- ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW);
+ if ((ret = ff_shuffle_filter_coefficients(c, c->hLumFilterPos, c->hLumFilterSize, c->hLumFilter, dstW)) < 0)
+ goto fail;
if ((ret = initFilter(&c->hChrFilter, &c->hChrFilterPos,
&c->hChrFilterSize, c->chrXInc,
c->chrSrcW, c->chrDstW, filterAlign, 1 << 14,
@@ -1846,7 +1852,8 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
get_local_pos(c, c->chrSrcHSubSample, c->src_h_chr_pos, 0),
get_local_pos(c, c->chrDstHSubSample, c->dst_h_chr_pos, 0))) < 0)
goto fail;
- ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW);
+ if ((ret = ff_shuffle_filter_coefficients(c, c->hChrFilterPos, c->hChrFilterSize, c->hChrFilter, c->chrDstW)) < 0)
+ goto fail;
}
} // initialize horizontal stuff
@@ -218,7 +218,8 @@ static void check_hscale(void)
ff_sws_init_scale(ctx);
memcpy(filterAvx2, filter, sizeof(uint16_t) * (SRC_PIXELS * MAX_FILTER_WIDTH + MAX_FILTER_WIDTH));
if ((cpu_flags & AV_CPU_FLAG_AVX2) && !(cpu_flags & AV_CPU_FLAG_SLOW_GATHER))
- ff_shuffle_filter_coefficients(ctx, filterPosAvx, width, filterAvx2, SRC_PIXELS);
+ if (ff_shuffle_filter_coefficients(ctx, filterPosAvx, width, filterAvx2, SRC_PIXELS) < 0)
+ fail();
if (check_func(ctx->hcScale, "hscale_%d_to_%d_width%d", ctx->srcBpc, ctx->dstBpc + 1, width)) {
memset(dst0, 0, SRC_PIXELS * sizeof(dst0[0]));
As the potential failure of the memory allocation, the return value of the av_malloc() could be NULL and be dereferenced on. Therefore it should be better to check it and return error if fails. Also, the callers of the ff_shuffle_filter_coefficients() should deal with the return value. Fixes: f900a19fa9 ("libswscale: Adds ff_hscale8to15_4_avx2 and ff_hscale8to15_X4_avx2 for all filter sizes.") Signed-off-by: Jiasheng Jiang <jiasheng@iscas.ac.cn> --- libswscale/swscale_internal.h | 2 +- libswscale/utils.c | 13 ++++++++++--- tests/checkasm/sw_scale.c | 3 ++- 3 files changed, 13 insertions(+), 5 deletions(-)